Merge branch 'master' of https://github.com/Microsoft/LightGBM into ponnuswami-dcgPositionLimit

Author: ashok-ponnuswami-msft

.ci/test.sh

@@ -96,13 +96,19 @@ if [[ $TASK == "swig" ]]; then
     exit 0
 fi
 
-conda install -q -y -n $CONDA_ENV cloudpickle dask distributed joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
+conda install -q -y -n $CONDA_ENV cloudpickle joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
 
 # graphviz must come from conda-forge to avoid this on some linux distros:
 # https://github.com/conda-forge/graphviz-feedstock/issues/18
+#
+# dask and distributed must come from conda-forge because they need to be kept
+# in sync and conda-forge packages are updated more quickly (automatically based
+# on pushes to PyPI)
 conda install -q -y \
     -n $CONDA_ENV \
     -c conda-forge \
+        'dask>=2021.3.0' \
+        'distributed>=2021.3.0' \
         python-graphviz \
         xorg-libxau
 

.ci/test_r_package.sh

@@ -20,6 +20,12 @@ export _R_CHECK_CRAN_INCOMING_REMOTE_=0
 # to catch extreme problems
 export _R_CHECK_PKG_SIZES_THRESHOLD_=60
 
+# don't fail builds for long-running examples unless they're very long.
+# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254.
+if [[ $R_BUILD_TYPE != "cran" ]]; then
+    export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30
+fi
+
 # Get details needed for installing R components
 R_MAJOR_VERSION=( ${R_VERSION//./ } )
 if [[ "${R_MAJOR_VERSION}" == "3" ]]; then

.ci/test_r_package_windows.ps1

@@ -28,6 +28,26 @@ function Run-R-Code-Redirect-Stderr {
   Rscript --vanilla -e $decorated_code
 }
 
+# Remove all items matching some pattern from PATH environment variable
+function Remove-From-Path {
+  param(
+    [string]$pattern_to_remove
+  )
+  $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';'
+}
+
+# remove some details that exist in the GitHub Actions images which might
+# cause conflicts with R and other components installed by this script
+$env:RTOOLS40_HOME = ""
+Remove-From-Path ".*chocolatey.*"
+Remove-From-Path ".*Chocolatey.*"
+Remove-From-Path ".*Git.*mingw64.*"
+Remove-From-Path ".*msys64.*"
+Remove-From-Path ".*rtools40.*"
+Remove-From-Path ".*Strawberry.*"
+
+Remove-Item C:\rtools40 -Force -Recurse -ErrorAction Ignore
+
 # Get details needed for installing R components
 #
 # NOTES:
@@ -72,6 +92,12 @@ $env:_R_CHECK_CRAN_INCOMING_REMOTE_ = 0
 # to catch extreme problems
 $env:_R_CHECK_PKG_SIZES_THRESHOLD_ = 60
 
+# don't fail builds for long-running examples unless they're very long.
+# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254.
+if ($env:R_BUILD_TYPE -ne "cran") {
+    $env:_R_CHECK_EXAMPLE_TIMING_THRESHOLD_ = 30
+}
+
 if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
   $env:CXX = "$env:RTOOLS_MINGW_BIN/g++.exe"
   $env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
@@ -92,7 +118,7 @@ Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT
 Write-Output "Done installing R"
 
 Write-Output "Installing Rtools"
-./Rtools.exe /VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH ; Check-Output $?
+Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH" ; Check-Output $?
 Write-Output "Done installing Rtools"
 
 Write-Output "Installing dependencies"

.github/CODEOWNERS

@@ -11,24 +11,24 @@
 
 # other catch-alls that will get matched if specific rules below are not matched
 *.R    @Laurae2 @jameslamb
-*.py    @StrikerRUS @chivee @wxchan @henry0312
-*.cpp    @guolinke @chivee @btrotta
-*.h    @guolinke @chivee @btrotta
+*.py    @StrikerRUS @chivee @wxchan @henry0312 @shiyu1994
+*.cpp    @guolinke @chivee @btrotta @shiyu1994
+*.h    @guolinke @chivee @btrotta @shiyu1994
 
 # main C++ code
-include/    @guolinke @chivee @btrotta
-src/    @guolinke @chivee @btrotta
-CMakeLists.txt    @guolinke @chivee @Laurae2 @jameslamb @wxchan @henry0312 @StrikerRUS @huanzhang12 @btrotta
-tests/c_api_test/    @guolinke @chivee @btrotta
-tests/cpp_test/    @guolinke @chivee @btrotta
-tests/data/    @guolinke @chivee @btrotta
-windows/    @guolinke @chivee @btrotta @StrikerRUS
+include/    @guolinke @chivee @btrotta @shiyu1994
+src/    @guolinke @chivee @btrotta @shiyu1994
+CMakeLists.txt    @guolinke @chivee @Laurae2 @jameslamb @wxchan @henry0312 @StrikerRUS @huanzhang12 @btrotta @shiyu1994
+tests/c_api_test/    @guolinke @chivee @btrotta @shiyu1994
+tests/cpp_test/    @guolinke @chivee @btrotta @shiyu1994
+tests/data/    @guolinke @chivee @btrotta @shiyu1994
+windows/    @guolinke @chivee @btrotta @StrikerRUS @shiyu1994
 
 # R code
 R-package/    @Laurae2 @jameslamb
 
 # Python code
-python-package/    @StrikerRUS @chivee @wxchan @henry0312
+python-package/    @StrikerRUS @chivee @wxchan @henry0312 @shiyu1994
 
 # Dask integration
 python-package/lightgbm/dask.py    @jameslamb
@@ -46,15 +46,15 @@ examples/     @StrikerRUS @jameslamb @guolinke
 
 # docker setup
 docker/    @StrikerRUS @jameslamb
-docker/dockerfile-cli    @guolinke @chivee
+docker/dockerfile-cli    @guolinke @chivee @shiyu1994
 docker/gpu/    @huanzhang12
-docker/dockerfile-python    @StrikerRUS @chivee @wxchan @henry0312
+docker/dockerfile-python    @StrikerRUS @chivee @wxchan @henry0312 @shiyu1994
 docker/dockerfile-r    @Laurae2 @jameslamb
 
 # GPU code
 docs/GPU-*.rst    @huanzhang12
-src/treelearner/gpu_tree_learner.cpp    @huanzhang12 @guolinke @chivee
-src/treelearner/tree_learner.cpp    @huanzhang12 @guolinke @chivee
+src/treelearner/gpu_tree_learner.cpp    @huanzhang12 @guolinke @chivee @shiyu1994
+src/treelearner/tree_learner.cpp    @huanzhang12 @guolinke @chivee @shiyu1994
 
 # JAVA code
-swig/    @imatiach-msft
+swig/    @guolinke @chivee @shiyu1994

.github/workflows/r_package.yml

@@ -60,6 +60,12 @@ jobs:
             compiler: clang
             r_version: 4.0
             build_type: cmake
+          - os: windows-latest
+            task: r-package
+            compiler: MINGW
+            toolchain: MINGW
+            r_version: 3.6
+            build_type: cmake
           - os: windows-latest
             task: r-package
             compiler: MINGW

.github/workflows/r_solaris.yml

@@ -31,6 +31,7 @@ jobs:
       - name: Send init status
         if: ${{ always() }}
         run: |
+          $GITHUB_WORKSPACE/.ci/set_commit_status.sh "${{ github.workflow }}" "pending" "${{ github.event.client_payload.pr_sha }}"
           $GITHUB_WORKSPACE/.ci/append_comment.sh \
             "${{ github.event.client_payload.comment_number }}" \
             "Workflow **${{ github.workflow }}** has been triggered! 🚀\r\n${GITHUB_SERVER_URL}/microsoft/LightGBM/actions/runs/${GITHUB_RUN_ID}"

.gitignore

@@ -269,6 +269,7 @@ _Pvt_Extensions
 *.app
 /windows/LightGBM.VC.db
 lightgbm
+/testlightgbm
 
 # Created by https://www.gitignore.io/api/python
 
@@ -354,6 +355,7 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+Untitled*.ipynb
 
 # pyenv
 .python-version

CMakeLists.txt

@@ -6,6 +6,12 @@ OPTION(USE_HDFS "Enable HDFS support (EXPERIMENTAL)" OFF)
 OPTION(USE_TIMETAG "Set to ON to output time costs" OFF)
 OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF)
 OPTION(USE_DEBUG "Set to ON for Debug mode" OFF)
+OPTION(USE_SANITIZER "Use santizer flags" OFF)
+SET(SANITIZER_PATH "" CACHE STRING "Path to sanitizer libs")
+SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING
+  "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
+address, leak, undefined and thread.")
+OPTION(BUILD_CPP_TEST "Build C++ tests with Google Test" OFF)
 OPTION(BUILD_STATIC_LIB "Build static library" OFF)
 OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF)
 OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF)
@@ -26,6 +32,14 @@ endif()
 
 PROJECT(lightgbm LANGUAGES C CXX)
 
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
+
+#-- Sanitizer
+if (USE_SANITIZER)
+  include(cmake/Sanitizer.cmake)
+  enable_sanitizers("${ENABLED_SANITIZERS}")
+endif (USE_SANITIZER)
+
 if(__INTEGRATE_OPENCL)
   set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE)
   set(USE_GPU OFF CACHE BOOL "" FORCE)
@@ -451,6 +465,25 @@ if(__BUILD_FOR_R)
   endif(MSVC)
 endif(__BUILD_FOR_R)
 
+#-- Google C++ tests
+if(BUILD_CPP_TEST)
+  find_package(GTest CONFIG)
+  if(NOT GTEST_FOUND)
+    message(STATUS "Did not find Google Test in the system root. Fetching Google Test now...")
+    include(FetchContent)
+    FetchContent_Declare(
+      googletest
+      GIT_REPOSITORY https://github.com/google/googletest.git
+      GIT_TAG        release-1.10.0
+    )
+    FetchContent_MakeAvailable(googletest)
+    add_library(GTest::GTest ALIAS gtest)
+  endif()
+  file(GLOB CPP_TEST_SOURCES tests/cpp_test/*.cpp)
+  add_executable(testlightgbm ${CPP_TEST_SOURCES} ${SOURCES})
+  target_link_libraries(testlightgbm PRIVATE GTest::GTest)
+endif()
+
 install(TARGETS lightgbm _lightgbm
         RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
         LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib

cmake/Sanitizer.cmake

@@ -0,0 +1,61 @@
+# Set appropriate compiler and linker flags for sanitizers.
+#
+# Usage of this module:
+#  enable_sanitizers("address;leak")
+
+# Add flags
+macro(enable_sanitizer sanitizer)
+  if(${sanitizer} MATCHES "address")
+    find_package(ASan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address")
+    link_libraries(${ASan_LIBRARY})
+
+  elseif(${sanitizer} MATCHES "thread")
+    find_package(TSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread")
+    link_libraries(${TSan_LIBRARY})
+
+  elseif(${sanitizer} MATCHES "leak")
+    find_package(LSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak")
+    link_libraries(${LSan_LIBRARY})
+
+  elseif(${sanitizer} MATCHES "undefined")
+    find_package(UBSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined")
+    link_libraries(${UBSan_LIBRARY})
+
+  else()
+    message(FATAL_ERROR "Santizer ${sanitizer} not supported.")
+  endif()
+endmacro()
+
+macro(enable_sanitizers SANITIZERS)
+  # Check sanitizers compatibility.
+  foreach ( _san ${SANITIZERS} )
+    string(TOLOWER ${_san} _san)
+    if (_san MATCHES "thread")
+      if (${_use_other_sanitizers})
+        message(FATAL_ERROR
+          "thread sanitizer is not compatible with ${_san} sanitizer.")
+      endif()
+      set(_use_thread_sanitizer 1)
+    else ()
+      if (${_use_thread_sanitizer})
+        message(FATAL_ERROR
+          "${_san} sanitizer is not compatible with thread sanitizer.")
+      endif()
+      set(_use_other_sanitizers 1)
+    endif()
+  endforeach()
+
+  message(STATUS "Sanitizers: ${SANITIZERS}")
+
+  foreach( _san ${SANITIZERS} )
+    string(TOLOWER ${_san} _san)
+    enable_sanitizer(${_san})
+  endforeach()
+  message(STATUS "Sanitizers compile flags: ${SAN_COMPILE_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_COMPILE_FLAGS}")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_COMPILE_FLAGS}")
+endmacro()

cmake/modules/FindASan.cmake

@@ -0,0 +1,9 @@
+set(ASan_LIB_NAME ASan)
+
+find_library(ASan_LIBRARY
+  NAMES libasan.so libasan.so.5 libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0 libasan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ASan DEFAULT_MSG
+  ASan_LIBRARY)

cmake/modules/FindLSan.cmake

@@ -0,0 +1,9 @@
+set(LSan_LIB_NAME lsan)
+
+find_library(LSan_LIBRARY
+  NAMES liblsan.so liblsan.so.0 liblsan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LSan DEFAULT_MSG
+  LSan_LIBRARY)

cmake/modules/FindTSan.cmake

@@ -0,0 +1,9 @@
+set(TSan_LIB_NAME tsan)
+
+find_library(TSan_LIBRARY
+  NAMES libtsan.so libtsan.so.0 libtsan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(TSan DEFAULT_MSG
+  TSan_LIBRARY)

cmake/modules/FindUBSan.cmake

@@ -0,0 +1,9 @@
+set(UBSan_LIB_NAME UBSan)
+
+find_library(UBSan_LIBRARY
+  NAMES libubsan.so libubsan.so.1 libubsan.so.0 libubsan.so.0.0.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(UBSan DEFAULT_MSG
+  UBSan_LIBRARY)

docs/FAQ.rst

@@ -22,6 +22,7 @@ You may also ping a member of the core team according to the relevant area of ex
 
 -  `@guolinke <https://github.com/guolinke>`__ **Guolin Ke** (C++ code / R-package / Python-package)
 -  `@chivee <https://github.com/chivee>`__ **Qiwei Ye** (C++ code / Python-package)
+-  `@shiyu1994 <https://github.com/shiyu1994>`__ **Yu Shi** (C++ code / Python-package)
 -  `@btrotta <https://github.com/btrotta>`__ **Belinda Trotta** (C++ code)
 -  `@Laurae2 <https://github.com/Laurae2>`__ **Damien Soukhavong** (R-package)
 -  `@jameslamb <https://github.com/jameslamb>`__ **James Lamb** (R-package / Dask-package)