Containers and build process improvements/fixes (#1602)

- Use ndk 21 due to Go bug: https://github.com/golang/go/issues/42655
- Use pre-built clang binaries instead of building ourselves
- Build syncthing when creating container to prepopulate modules

docker/Dockerfile

@@ -20,16 +20,19 @@ RUN mkdir -p /opt/android-sdk && cd /opt/android-sdk && \
 ENV ANDROID_HOME /opt/android-sdk
 
 # Accept the SDK license, as we can't install packages otherwise
-RUN yes | ${ANDROID_HOME}/tools/bin/sdkmanager --licenses
+RUN yes | ${ANDROID_HOME}/tools/bin/sdkmanager --licenses > /dev/null
+
+# NDK version (r22 fails to build)
+ENV NDK_VERSION 21.4.7075529
 
 # Install other android packages, including NDK
-RUN ${ANDROID_HOME}/tools/bin/sdkmanager tools platform-tools "build-tools;29.0.3" "platforms;android-29" "extras;android;m2repository" ndk-bundle
+RUN ${ANDROID_HOME}/tools/bin/sdkmanager tools platform-tools "build-tools;29.0.3" "platforms;android-29" "extras;android;m2repository" "ndk;${NDK_VERSION}"
 
 # Accept licenses of newly installed packages
 RUN yes | ${ANDROID_HOME}/tools/bin/sdkmanager --licenses
 
 # Setup the NDK path
-ENV ANDROID_NDK_HOME ${ANDROID_HOME}/ndk-bundle
+ENV ANDROID_NDK_HOME ${ANDROID_HOME}/ndk/${NDK_VERSION}
 
 # Enable prebuild mode
 ENV SYNCTHING_ANDROID_PREBUILT 1

docker/prebuild.sh

@@ -2,45 +2,8 @@
 
 [ -z "$SYNCTHING_ANDROID_PREBUILT" ] && echo "Prebuild disabled" && exit 0
 
-for ARCH in arm x86 arm64 x86_64; do
-  GOARCH=${ARCH}
-  SDK=16
-  # The values here must correspond with those in ../syncthing/build-syncthing.py
-  case ${ARCH} in
-      arm)
-        GCC="arm-linux-androideabi-clang"
-        ;;
-      arm64)
-        SDK=21
-        GCC="aarch64-linux-android-clang"
-        ;;
-      x86)
-        GOARCH=386
-        GCC="i686-linux-android-clang"
-        ;;
-      x86_64)
-        SDK=21
-        GOARCH=amd64
-        GCC="x86_64-linux-android21-clang"
-        ;;
-      *)
-        echo "Invalid architecture"
-        exit 1
-  esac
-
-  STANDALONE_NDK_DIR="${ANDROID_NDK_HOME}/standalone-ndk/android-${SDK}-${GOARCH}"
-  echo "Building standalone NDK - ${STANDALONE_NDK_DIR}"
-  ${ANDROID_NDK_HOME}/build/tools/make-standalone-toolchain.sh \
-    --platform=android-${SDK} --arch=${ARCH} \
-    --install-dir=${STANDALONE_NDK_DIR}
-
-   echo "Pre-building Go standard library for $GOARCH"
-   CGO_ENABLED=1 CC="${STANDALONE_NDK_DIR}/bin/${GCC}" \
-      GOOS=android GOARCH=$GOARCH go install -v std
-done
-
 echo "Prepopulating gradle and go build/pkg cache"
-git clone https://github.com/syncthing/syncthing-android
+git clone --recurse-submodules https://github.com/syncthing/syncthing-android
 cd syncthing-android
 ./gradlew --no-daemon lint buildNative
 cd ..

syncthing/build-syncthing.py

@@ -13,20 +13,20 @@ BUILD_TARGETS = [
         'arch': 'arm',
         'goarch': 'arm',
         'jni_dir': 'armeabi',
-        'cc': 'arm-linux-androideabi-clang',
+        'cc': 'armv7a-linux-androideabi16-clang',
     },
     {
         'arch': 'arm64',
         'goarch': 'arm64',
         'jni_dir': 'arm64-v8a',
-        'cc': 'aarch64-linux-android-clang',
+        'cc': 'aarch64-linux-android21-clang',
         'min_sdk': 21,
     },
     {
         'arch': 'x86',
         'goarch': '386',
         'jni_dir': 'x86',
-        'cc': 'i686-linux-android-clang',
+        'cc': 'i686-linux-android16-clang',
     },
     {
         'arch': 'x86_64',
@@ -37,7 +37,6 @@ BUILD_TARGETS = [
     }
 ]
 
-
 def fail(message, *args, **kwargs):
     print((message % args).format(**kwargs))
     sys.exit(1)
@@ -83,37 +82,8 @@ subprocess.check_call([
 
 for target in BUILD_TARGETS:
     target_min_sdk = str(target.get('min_sdk', min_sdk))
-    print('Building for', target['arch'])
 
-    if os.environ.get('SYNCTHING_ANDROID_PREBUILT', ''):
+    print('Building syncthing for', target['arch'])
-        # The environment variable indicates the SDK and stdlib was prebuilt, set a custom paths.
-        standalone_ndk_dir = '%s/standalone-ndk/android-%s-%s' % (
-            get_ndk_home(), target_min_sdk, target['goarch']
-        )
-        pkg_argument = []
-    else:
-        # Build standalone NDK toolchain if it doesn't exist.
-        # https://developer.android.com/ndk/guides/standalone_toolchain.html
-        standalone_ndk_dir = '%s/standalone-ndk/android-%s-%s' % (
-            build_dir, target_min_sdk, target['goarch']
-        )
-        pkg_argument = ['-pkgdir', os.path.join(go_build_dir, target['goarch'])]
-
-    if not os.path.isdir(standalone_ndk_dir):
-        print('Building standalone NDK for', target['arch'], 'API level', target_min_sdk, 'to', standalone_ndk_dir)
-        subprocess.check_call([
-            sys.executable,
-            os.path.join(get_ndk_home(), 'build', 'tools', 'make_standalone_toolchain.py'),
-            '--arch',
-            target['arch'],
-            '--api',
-            target_min_sdk,
-            '--install-dir',
-            standalone_ndk_dir,
-            '-v'
-        ])
-
-    print('Building syncthing')
 
     environ = os.environ.copy()
     environ.update({
@@ -121,9 +91,15 @@ for target in BUILD_TARGETS:
         'CGO_ENABLED': '1',
     })
 
-    subprocess.check_call([
+    cc = '/'.join([
-        'go', 'run', 'build.go', '-goos', 'android', '-goarch', target['goarch'], '-cc', os.path.join(standalone_ndk_dir, 'bin', target['cc'])
+        get_ndk_home(), "toolchains/llvm/prebuilt/linux-x86_64", "bin",
-    ] + pkg_argument + ['-no-upgrade', 'build'], env=environ, cwd=syncthing_dir)
+        target['cc']])
+    subprocess.check_call(
+        ['go', 'run', 'build.go', '-goos', 'android',
+         '-goarch', target['goarch'], '-cc', cc,
+         '-pkgdir', os.path.join(go_build_dir, target['goarch']),
+         '-no-upgrade', 'build'],
+        env=environ, cwd=syncthing_dir)
 
     # Copy compiled binary to jniLibs folder
     target_dir = os.path.join(project_dir, 'app', 'src', 'main', 'jniLibs', target['jni_dir'])