#################################################################
#  CMake configure file for Monte Carlo eXtreme (mmc)
#  Qianqian Fang <q.fang at neu.edu>
#  2018/08/26
#################################################################
# Note: when multiple gcc versions are installed, cmake may find
# the highest version, but nvcc may throw an error saying it does
# not support this gcc version. To solve this, one should run
#      CC=gcc-X CXX=g++-X cmake ..
# where "X" is the version (such as 9) that both installed on
# your system and is supported by nvcc
#################################################################

cmake_minimum_required(VERSION 3.3)

project(mmc)

if(${CMAKE_VERSION} VERSION_GREATER "3.11.20")
    cmake_policy(SET CMP0074 NEW)  # to make -DOPENCL_ROOT= work with newer CMake versions as well
endif()

find_package(OpenMP REQUIRED)
find_package(OpenCL REQUIRED)

add_subdirectory(zmat)

option(BUILD_MEX "Build mex" ON)
option(BUILD_CUDA "Build cuda" OFF)
option(BUILD_OPTIX "Build _rtmmc, pmmc with the CUDA and OptiX backends (needs the OptiX headers in OPTIX_ROOT)" OFF)

if(BUILD_OPTIX)
    if(NOT OPTIX_ROOT)
        set(OPTIX_ROOT "$ENV{OPTIX_ROOT}")
    endif()
    if(NOT EXISTS "${OPTIX_ROOT}/include/optix.h")
        message(FATAL_ERROR "BUILD_OPTIX needs OPTIX_ROOT, a folder with include/optix.h")
    endif()
endif()

if(BUILD_PYTHON)
    add_subdirectory(pybind11)
    find_package (Python3 COMPONENTS Interpreter Development)
    include_directories(${PYTHON_INCLUDE_DIRS})
endif()

if(BUILD_MEX)
    find_package(Matlab)
endif()

if(BUILD_CUDA OR BUILD_OPTIX)
  find_package(CUDA QUIET REQUIRED)
endif()

# C Options
if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
    set(CMAKE_CXX_FLAGS "-Wall -g -DMCX_EMBED_CL -fno-strict-aliasing -m64 -O3 -DUSE_OS_TIMER -DUSE_OPENCL -DMMC_XORSHIFT -D_hypot=hypot -fPIC ${OpenMP_CXX_FLAGS}")
else()
    set(CMAKE_CXX_FLAGS "-Wall -g -DMCX_EMBED_CL -fno-strict-aliasing -m64 -DMMC_USE_SSE -DHAVE_SSE2 -msse -msse2 -msse3 -mssse3 -msse4.1 -O3 -DUSE_OS_TIMER -DUSE_OPENCL -DMMC_XORSHIFT -D_hypot=hypot -fPIC ${OpenMP_CXX_FLAGS}")
endif()

# Apple clang's OpenMP flags do not link a runtime: link Homebrew's static libomp into modules
# (_pmmc) on both arm64 and x86_64, so the wheels need no libomp.dylib; without it the x86_64
# _pmmc failed to import with "symbol not found: ___kmpc_barrier"
if(APPLE)
    find_library(LIBOMP_STATIC NAMES libomp.a HINTS /opt/homebrew/opt/libomp/lib /usr/local/opt/libomp/lib /opt/homebrew/lib /usr/local/lib)
    if(LIBOMP_STATIC)
        set(CMAKE_MODULE_LINKER_FLAGS "${LIBOMP_STATIC}")
    endif()
endif()
set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/../bin)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/../lib)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)

set(RESOURCE_COMPILER xxd)
set(RESOURCE_INPUT "mmc_core.cl")
set(RESOURCE_OUTPUT "${CMAKE_SOURCE_DIR}/mmc_core.clh")

add_custom_target(
    clheader
    COMMAND bash -c "cd ${CMAKE_SOURCE_DIR}; ${RESOURCE_COMPILER} -i ${RESOURCE_INPUT} | sed 's/\\([0-9a-f]\\)$/\\0, 0x00/' > ${RESOURCE_OUTPUT}"
    comment "Compiling ${RESOURCE_INPUT} to ${RESOURCE_OUTPUT}"
    VERBATIM
)

if(BUILD_CUDA OR BUILD_OPTIX)
  # NVC# NVCC Options
  if(MSVC)
    set(CUDA_HOST_FLAGS -Xcompiler "${OpenMP_CXX_FLAGS}" -D_USE_MATH_DEFINES)
  else()
    set(CUDA_HOST_FLAGS -Xcompiler -fPIC -Xcompiler -Wall -Xcompiler "${OpenMP_CXX_FLAGS}")
  endif()
  set(
      CUDA_NVCC_FLAGS
      ${CUDA_NVCC_FLAGS};
      -g -lineinfo -DMCX_SAVE_DETECTORS -DMCX_DO_REFLECTION -DUSE_DMMC -DUSE_BLBADOUEL -use_fast_math -Xptxas -O3,-v ${CUDA_HOST_FLAGS}
      )

  # a fat binary, as the Makefile builds: native code from Maxwell (sm_52) to Ampere (sm_86) and
  # the PTX of the newest for later GPUs; PTX of a pre-Volta GPU runs 2-3x slower on newer ones
  set(CUDA_ARCHS "52;60;70;75;80;86" CACHE STRING "GPU architectures (sm_XX) to build the CUDA backend for")
  execute_process(COMMAND ${CUDA_NVCC_EXECUTABLE} --list-gpu-code OUTPUT_VARIABLE NVCC_GPU_CODES ERROR_QUIET)
  set(CUDA_SMS "")
  foreach(sm ${CUDA_ARCHS})
    if(NVCC_GPU_CODES MATCHES "sm_${sm}")
      list(APPEND CUDA_SMS ${sm})
      list(APPEND CUDA_NVCC_FLAGS -gencode=arch=compute_${sm},code=sm_${sm})
    endif()
  endforeach()
  if(CUDA_SMS)
    list(GET CUDA_SMS -1 lastsm)
    list(APPEND CUDA_NVCC_FLAGS -gencode=arch=compute_${lastsm},code=compute_${lastsm})
  endif()
endif()

# Add include directories
include_directories(cjson ubj zmat zmat/easylzma)

# Add link directories
link_directories(zmat)

# Create mmc library
add_library(mmc STATIC
    mmc_utils.c
    mmc_utils.h
    mmc_mesh.c
    mmc_mesh.h
    mmc_host.c
    mmc_host.h
    mmc_raytrace.c
    mmc_raytrace.h
    mmc_rand_xorshift128p.h
    mmc_bench.h
    mmc_bench.c
    mmc_tictoc.c
    mmc_tictoc.h
    mmc_neurojson.cpp
    mmc_neurojson.h
    mmc_cl_utils.c
    mmc_cl_utils.h
    mmc_cl_host.c
    mmc_cl_host.h
    mmc_highorder.cpp
    cjson/cJSON.c
    cjson/cJSON.h
    ubj/ubj.h
    ubj/ubjw.c
    )

# Add all project units
add_executable(
    mmc-exe
    mmc.c
    )

set_target_properties(mmc-exe
        PROPERTIES OUTPUT_NAME mmc)

# Link options
target_link_libraries(
    mmc-exe
    mmc OpenMP::OpenMP_CXX
    zmat
    m
    OpenCL::OpenCL
    )

add_dependencies(mmc clheader)
add_dependencies(mmc-exe clheader)

#static link libraries
set_target_properties(mmc-exe PROPERTIES LINK_SEARCH_START_STATIC 1)
set_target_properties(mmc-exe PROPERTIES LINK_SEARCH_END_STATIC 1)
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")

if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
    set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
    set(CMAKE_SHARED_LINKER_FLAGS "-static-libgcc -static-libstdc++")
    set(CMAKE_MODULE_LINKER_FLAGS "-static-libgcc -static-libstdc++")

    if (UNIX AND NOT APPLE)
        # do not export the symbols of the static libstdc++/libgcc: _pmmc exported their GNU_UNIQUE
        # symbols (std::locale facet ids), which the dynamic linker merges process-wide, so a module
        # using the system libstdc++ (such as _pmcx) crashed in std::cout after _pmmc was loaded
        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--exclude-libs,ALL")
        set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--exclude-libs,ALL")
    endif()
endif()

if(BUILD_CUDA)
    # Create mmc library
    cuda_add_library(mmc-cuda STATIC
        mmc_cu_host.cu
        mmc_cu_host.h
        mmc_utils.c
        mmc_utils.h
        )

  # Create mmc-trinity binary
  cuda_add_executable(
      mmc-trinity
      mmc.c
      )
  # Link options
  target_link_libraries(
      mmc-trinity
      mmc mmc-cuda OpenMP::OpenMP_CXX
      zmat
      m
      OpenCL::OpenCL
      )

  target_compile_definitions(mmc-trinity PUBLIC USE_CUDA USE_BLBADOUEL MCX_SAVE_DETECTORS MCX_DO_REFLECTION USE_DMMC)
  add_dependencies(mmc-trinity mmc mmc-cuda)

  # Create mmc library
  cuda_add_library(mmclab-cudahost STATIC
        mmc_cu_host.cu
        mmc_cu_host.h
        mmc_utils.c
        mmc_utils.h
        )
  target_compile_definitions(mmclab-cudahost PUBLIC USE_CUDA USE_BLBADOUEL MCX_SAVE_DETECTORS MCX_DO_REFLECTION USE_DMMC MCX_CONTAINER)

  set_target_properties(mmc-trinity
        PROPERTIES OUTPUT_NAME mmciii)
  #static link libraries
  #set_target_properties(mmc-trinity PROPERTIES LINK_SEARCH_START_STATIC 1)
  #set_target_properties(mmc-trinity PROPERTIES LINK_SEARCH_END_STATIC 1)
endif()

# Build mex file
if(BUILD_PYTHON)
    # Create mcx-matlab library
    set(PMMC_SOURCES
            mmc_utils.c
            mmc_utils.h
            mmc_mesh.c
            mmc_mesh.h
            mmc_host.c
            mmc_host.h
            mmc_raytrace.c
            mmc_raytrace.h
            mmc_rand_xorshift128p.h
            mmc_bench.h
            mmc_bench.c
            mmc_tictoc.c
            mmc_tictoc.h
            mmc_cl_utils.c
            mmc_cl_utils.h
            mmc_cl_host.c
            mmc_cl_host.h
            mmc_highorder.cpp
            cjson/cJSON.c
            cjson/cJSON.h
            ubj/ubj.h
            ubj/ubjw.c
            pmmc.cpp
            )
    add_library(_pmmc MODULE ${PMMC_SOURCES})

    target_compile_definitions(_pmmc PUBLIC MCX_CONTAINER PYBIND11_VERSION_MAJOR)

    target_link_libraries(_pmmc pybind11::module pybind11::lto pybind11::windows_extras OpenCL::OpenCL)

    pybind11_extension(_pmmc)
    pybind11_strip(_pmmc)

    add_dependencies(_pmmc clheader)
    set_target_properties(_pmmc PROPERTIES CXX_VISIBILITY_PRESET "hidden")

    # opt-in CUDA backend (compute='cuda'), e.g. CMAKE_ARGS="-DBUILD_CUDA=ON" pip install .
    if(BUILD_CUDA)
        # FindCUDA hands nvcc no target definitions: MCX_CONTAINER (no output files; mmc_utils.c
        # leaves out mcx_savedetphoton) goes in by OPTIONS, or the module fails to load
        cuda_add_library(pmmc-cudahost STATIC
            mmc_cu_host.cu
            mmc_cu_host.h
            OPTIONS -DMCX_CONTAINER -DPYBIND11_VERSION_MAJOR
            )
        target_compile_definitions(_pmmc PUBLIC USE_CUDA)
        target_link_libraries(_pmmc pmmc-cudahost ${CUDA_LIBRARIES})
        add_dependencies(pmmc-cudahost clheader)
    endif()

    # _rtmmc: the same module with the CUDA (compute='cuda') and OptiX (compute='optix') backends,
    # for the mmc-trinity packages, e.g. CMAKE_ARGS="-DBUILD_OPTIX=ON<tab>-DOPTIX_ROOT=/path/to/optix";
    # _pmmc stays the CPU/OpenCL module of the PyPI wheels, and pmmc.run picks _rtmmc for cuda/optix
    if(BUILD_OPTIX)
        # the CUDA code, built for every architecture, does not depend on Python: a wheel build
        # for each Python version may link the one built first, given by RTMMC_CUDAHOST_LIB
        if(RTMMC_CUDAHOST_LIB)
            add_library(rtmmc-cudahost STATIC IMPORTED)
            set_target_properties(rtmmc-cudahost PROPERTIES IMPORTED_LOCATION ${RTMMC_CUDAHOST_LIB})
        else()
            cuda_add_library(rtmmc-cudahost STATIC
                mmc_cu_host.cu
                mmc_cu_host.h
                OPTIONS -DMCX_CONTAINER -DPYBIND11_VERSION_MAJOR
                )
            add_dependencies(rtmmc-cudahost clheader)
        endif()

        # the OptiX programs are PTX that OptiX compiles at run time for the GPU, embedded in
        # mmc_optix_utils.cpp: by incbin from built/mmc_optix_core.ptx (relative to the build
        # folder, where the compiler runs), or with MSVC from the char array of xxd -i
        set(OPTIX_PTX "${CMAKE_BINARY_DIR}/built/mmc_optix_core.ptx")
        set(OPTIX_PTX_DEFS -DUSE_OS_TIMER -DUSE_OPENCL -DMMC_XORSHIFT -DUSE_CUDA -DUSE_OPTIX -DUSE_CUDA_VECTOR_TYPES)
        add_custom_command(
            OUTPUT ${OPTIX_PTX} ${CMAKE_BINARY_DIR}/built/mmc_optix_core_ptx.h
            COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/built
            COMMAND ${CUDA_NVCC_EXECUTABLE} -O3 -ptx --expt-relaxed-constexpr -use_fast_math -m64 ${OPTIX_PTX_DEFS}
                    -I${CUDA_INCLUDE_DIRS} -I${OPTIX_ROOT}/include -o ${OPTIX_PTX} ${CMAKE_SOURCE_DIR}/mmc_optix_core.cu
            COMMAND bash -c "cd ${CMAKE_BINARY_DIR}/built && ${RESOURCE_COMPILER} -i mmc_optix_core.ptx | sed 's/\\([0-9a-f]\\)$/\\0, 0x00/' > mmc_optix_core_ptx.h"
            DEPENDS ${CMAKE_SOURCE_DIR}/mmc_optix_core.cu
            VERBATIM
        )
        add_custom_target(optixptx DEPENDS ${OPTIX_PTX})
        add_library(_rtmmc MODULE ${PMMC_SOURCES} mmc_optix_utils.cpp mmc_optix_host.cpp)
        set_source_files_properties(mmc_optix_utils.cpp PROPERTIES OBJECT_DEPENDS ${OPTIX_PTX})
        if(NOT MSVC)
            # the incbin assembly does not survive pybind11's link-time optimization (gas: "leb128
            # operand is an undefined symbol"), so this file is compiled to a plain object
            set_source_files_properties(mmc_optix_utils.cpp PROPERTIES COMPILE_OPTIONS -fno-lto)
        endif()
        target_compile_definitions(_rtmmc PUBLIC MCX_CONTAINER PYBIND11_VERSION_MAJOR USE_CUDA USE_OPTIX USE_CUDA_VECTOR_TYPES PMMC_MODULE_NAME=_rtmmc)
        target_include_directories(_rtmmc PRIVATE ${CUDA_INCLUDE_DIRS} ${OPTIX_ROOT}/include ${CMAKE_BINARY_DIR}/built)
        # the driver, libcuda.so.1: FindCUDA misses it in NVIDIA's redistributable archives, which
        # hold only the stub (lib/stubs), and CMAKE_FIND_LIBRARY_SUFFIXES above finds only .a files
        set(CUDA_DRIVER_LIB ${CUDA_CUDA_LIBRARY})
        if(NOT CUDA_DRIVER_LIB)
            set(CUDA_DRIVER_LIB -L${CUDA_TOOLKIT_ROOT_DIR}/lib/stubs -L${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs cuda)
        endif()
        target_link_libraries(_rtmmc pybind11::module pybind11::lto pybind11::windows_extras OpenCL::OpenCL
                              rtmmc-cudahost ${CUDA_LIBRARIES} ${CUDA_DRIVER_LIB} ${CMAKE_DL_LIBS})
        pybind11_extension(_rtmmc)
        pybind11_strip(_rtmmc)
        add_dependencies(_rtmmc clheader optixptx)
        set_target_properties(_rtmmc PROPERTIES CXX_VISIBILITY_PRESET "hidden")
    endif()

endif()


# Build mex file
if(BUILD_MEX AND Matlab_FOUND)
    # Create mcx-matlab library
    add_library(mmc-matlab STATIC
            mmc_utils.c
            mmc_utils.h
            mmc_mesh.c
            mmc_mesh.h
            mmc_host.c
            mmc_host.h
            mmc_raytrace.c
            mmc_raytrace.h
            mmc_rand_xorshift128p.h
            mmc_bench.h
            mmc_bench.c
            mmc_tictoc.c
            mmc_tictoc.h
            mmc_cl_utils.c
            mmc_cl_utils.h
            mmc_cl_host.c
            mmc_cl_host.h
            mmc_highorder.cpp
            cjson/cJSON.c
            cjson/cJSON.h
            ubj/ubj.h
            ubj/ubjw.c
            )

    target_compile_definitions(mmc-matlab PUBLIC MCX_CONTAINER MATLAB_MEX_FILE)

    if(${CMAKE_VERSION} VERSION_LESS "3.24.0")
            matlab_add_mex(
              NAME mmclab
              SRC mmclab.cpp
              LINK_TO mmc-matlab OpenMP::OpenMP_CXX OpenCL::OpenCL zmat
            )
    else()
            matlab_add_mex(
              NAME mmclab
              SRC mmclab.cpp
              NO_IMPLICIT_LINK_TO_MATLAB_LIBRARIES
              LINK_TO ${Matlab_MEX_LIBRARY} ${Matlab_MX_LIBRARY} mmc-matlab OpenMP::OpenMP_CXX OpenCL::OpenCL zmat
            )
    endif()

    target_compile_definitions(mmclab PUBLIC MCX_CONTAINER MATLAB_MEX_FILE)

    set_target_properties(mmclab
            PROPERTIES OUTPUT_NAME ${CMAKE_SOURCE_DIR}/../mmclab/mmc)
    add_dependencies(mmclab clheader)

    if(BUILD_CUDA)
    # Create mcx-matlab library
        add_library(mmclab-cuda STATIC
                mmc_utils.c
                mmc_utils.h
                mmc_mesh.c
                mmc_mesh.h
                mmc_host.c
                mmc_host.h
                mmc_raytrace.c
                mmc_raytrace.h
                mmc_rand_xorshift128p.h
                mmc_bench.h
                mmc_bench.c
                mmc_tictoc.c
                mmc_tictoc.h
                mmc_neurojson.cpp
                mmc_neurojson.h
                mmc_cl_utils.c
                mmc_cl_utils.h
                mmc_cl_host.c
                mmc_cl_host.h
                mmc_highorder.cpp
                )
        target_compile_definitions(mmclab-cuda PUBLIC MCX_CONTAINER MATLAB_MEX_FILE USE_CUDA USE_OS_TIMER MMC_XORSHIFT MMC_USE_SSE HAVE_SSE2)

        if(${CMAKE_VERSION} VERSION_LESS "3.24.0")
                matlab_add_mex(
                  NAME mmclab-trinity
                  SRC mmclab.cpp
                  LINK_TO mmclab-cudahost mmclab-cuda OpenMP::OpenMP_CXX OpenCL::OpenCL zmat
                )
        else()
                matlab_add_mex(
                  NAME mmclab-trinity
                  SRC mmclab.cpp
                  NO_IMPLICIT_LINK_TO_MATLAB_LIBRARIES
                  LINK_TO ${Matlab_MEX_LIBRARY} ${Matlab_MX_LIBRARY} mmclab-cudahost mmclab-cuda OpenMP::OpenMP_CXX OpenCL::OpenCL zmat
                )
        endif()

        target_compile_definitions(mmclab-trinity PUBLIC MCX_CONTAINER MATLAB_MEX_FILE USE_CUDA)

        set_target_properties(mmclab-trinity
                PROPERTIES OUTPUT_NAME ${CMAKE_SOURCE_DIR}/../mmclab/mmciii)
        add_dependencies(mmclab-trinity clheader)
    endif()


endif()
