Android: restructure android search paths

1. Set CMAKE_FIND_ROOT_PATH unconditionally.

Revise the implementation from commit a7f41a7ee4 (Android: Fix find_*
search order within NDK for unified toolchains, 2020-10-13).  In the old
implementation, if people set CMAKE_FIND_ROOT_PATH, CMAKE_ANDROID_NDK
won't be added to find root. And all paths added to CMAKE_SYSTEM_*_PATH
below will be rerooted to the user specified root.

2. Add api level specific library path to CMAKE_SYSTEM_PREFIX_PATH.

As the discussion in [1], some people want the paths added by
UnixPaths.cmake. They install their libraries according to
GNUInstallDirs [2].

As a result, we cannot clear CMAKE_SYSTEM_PREFIX_PATH. It includes /usr
so no matter what we specify in CMAKE_SYSTEM_LIBRARY_PATH,
/usr/lib/<arch> will be searched first.

The author also pointed out a way to solve this issue [3]. In addition
to other paths, CMake also searches <root>/<prefix> [4]. So we can add
the API specific lib path to the beginning of CMAKE_SYSTEM_PREFIX_PATH,
to have it searched first.

[1] https://android-review.googlesource.com/c/platform/ndk/+/1486800
[2] https://cmake.org/cmake/help/latest/module/GNUInstallDirs.html
[3] https://github.com/android/ndk/issues/1179#issuecomment-613435081
[4] 11425041f0/Source/cmSearchPath.cxx (L202)

Modules/Platform/Android.cmake

@@ -37,14 +37,14 @@ endif()
 
 if(CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED)
   # Tell CMake not to search host sysroots for headers/libraries.
-  # CMAKE_FIND_ROOT_PATH must be non-empty for CMAKE_FIND_ROOT_PATH_MODE_* == ONLY
-  # to be meaningful.  The actual path  used here is fairly meaningless since CMake
-  # doesn't handle the NDK sysroot layout (per-arch and per-verion subdirectories for
-  # libraries), so find_library is handled separately by CMAKE_SYSTEM_LIBRARY_PATH.
-  # https://github.com/android-ndk/ndk/issues/890
-  if(NOT CMAKE_FIND_ROOT_PATH)
-    list(APPEND CMAKE_FIND_ROOT_PATH "${CMAKE_ANDROID_NDK}")
-  endif()
+
+  # All paths added to CMAKE_SYSTEM_*_PATH below will be rerooted under
+  # CMAKE_FIND_ROOT_PATH. This is set because:
+  # 1. Users may structure their libraries in a way similar to NDK. When they do that,
+  #    they can simply append another path to CMAKE_FIND_ROOT_PATH.
+  # 2. CMAKE_FIND_ROOT_PATH must be non-empty for CMAKE_FIND_ROOT_PATH_MODE_* == ONLY
+  #    to be meaningful. https://github.com/android-ndk/ndk/issues/890
+  list(APPEND CMAKE_FIND_ROOT_PATH "${CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED}/sysroot")
 
   # Allow users to override these values in case they want more strict behaviors.
   # For example, they may want to prevent the NDK's libz from being picked up so
@@ -66,20 +66,6 @@ if(CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED)
     set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
   endif()
 
-  # find_library's default search paths below a prefix do not match the Android
-  # sysroot layout, so we need to give the direct path to the libraries
-  # via CMAKE_SYSTEM_*_PATH.
-  #
-  # Ideally we'd set CMAKE_SYSTEM_PREFIX_PATH. But that causes the
-  # non-api-level-specific path to be searched first for find_library, which will
-  # cause libdl.a to be found before libdl.so.
-  # https://github.com/android/ndk/issues/929
-
-  # Clears the paths set by UnixPaths.cmake.
-  set(CMAKE_SYSTEM_PREFIX_PATH)
-  set(CMAKE_SYSTEM_INCLUDE_PATH)
-  set(CMAKE_SYSTEM_LIBRARY_PATH)
-
   # Don't search paths in PATH environment variable.
   if(NOT DEFINED CMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH)
     set(CMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH OFF)
@@ -88,17 +74,13 @@ if(CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED)
   # Allows CMake to find headers in the architecture-specific include directories.
   set(CMAKE_LIBRARY_ARCHITECTURE "${CMAKE_ANDROID_ARCH_TRIPLE}")
 
-  set(_ANDROID_SYSROOT_PREFIX "${CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED}/sysroot/usr")
-
-  list(APPEND CMAKE_SYSTEM_INCLUDE_PATH
-    "${_ANDROID_SYSROOT_PREFIX}/include/${CMAKE_LIBRARY_ARCHITECTURE}")
-  list(APPEND CMAKE_SYSTEM_INCLUDE_PATH "${_ANDROID_SYSROOT_PREFIX}/include")
-
   # Instructs CMake to search the correct API level for libraries.
-  list(APPEND CMAKE_SYSTEM_LIBRARY_PATH
-    "${_ANDROID_SYSROOT_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_SYSTEM_VERSION}")
-  list(APPEND CMAKE_SYSTEM_LIBRARY_PATH
-    "${_ANDROID_SYSROOT_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}")
+  # Besides the paths like <root>/<prefix>/lib/<arch>, cmake also searches <root>/<prefix>.
+  # So we can add the API level specific directory directly.
+  # https://github.com/android/ndk/issues/929
+  list(PREPEND CMAKE_SYSTEM_PREFIX_PATH
+    "/usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_SYSTEM_VERSION}"
+    )
 
   list(APPEND CMAKE_SYSTEM_PROGRAM_PATH "${CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED}/bin")
 endif()

Tests/RunCMake/Android/RunCMakeTest.cmake

@@ -103,6 +103,13 @@ foreach(ndk IN LISTS TEST_ANDROID_NDK)
     set(ndk_arg)
   endif()
 
+  set(RunCMake_TEST_OPTIONS
+    -DCMAKE_SYSTEM_NAME=Android
+    -DCMAKE_FIND_ROOT_PATH=/tmp
+    ${ndk_arg}
+    )
+  run_cmake(ndk-search-order)
+
   # Test failure cases.
   message(STATUS "ndk='${ndk}'")
   if(RunCMake_GENERATOR MATCHES "Visual Studio")

Tests/RunCMake/Android/ndk-search-order.cmake

@@ -0,0 +1,17 @@
+if(NOT CMAKE_ANDROID_NDK_TOOLCHAIN_UNIFIED)
+  return()
+endif()
+
+find_library(LIBDL dl)
+if(NOT LIBDL)
+  message(FATAL_ERROR "libdl not found.")
+endif()
+
+if(LIBDL MATCHES ".a$")
+  message(FATAL_ERROR "found libdl.a")
+endif()
+
+find_program(CLANG clang)
+if(NOT CLANG)
+  message(FATAL_ERROR "clang not found")
+endif()