Build Syncthing with CGO and target android (fixes #873)

.gitignore

@@ -15,6 +15,7 @@ gen/
 src/main/jniLibs/
 obj/
 .gradle/
+gobuild/
 
 # Local configuration file (sdk path, etc)
 local.properties

.gitmodules

@@ -1,12 +1,3 @@
-[submodule "ext/syncthing/src/github.com/syncthing/syncthing"]
+[submodule "syncthing"]
-	path = ext/syncthing/src/github.com/syncthing/syncthing
+	path = go/src/github.com/syncthing/syncthing
 	url = https://github.com/syncthing/syncthing.git
-	ignore = dirty
-[submodule "ext/golang/go"]
-	path = ext/golang/go
-	url = https://github.com/golang/go.git
-	ignore = dirty
-[submodule "ext/golang/go1.4"]
-	path = ext/golang/go1.4
-	url = https://github.com/golang/go.git
-	ignore = dirty

.travis.yml

@@ -1,6 +1,7 @@
 sudo: required
 language: android
 jdk: oraclejdk8
+dist: trusty
 
 # Install Android SDK
 android:
@@ -13,10 +14,18 @@ android:
     - android-26
     - extra-android-m2repository
 
+# Install Android NDK (apparently there is no easier way to do this)
+# https://github.com/travis-ci/travis-ci/issues/5395
+before_script:
+  - curl -L https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip -O
+  - unzip -q android-ndk-r15c-linux-x86_64.zip
+  - export ANDROID_NDK_HOME=`pwd`/android-ndk-r15c
+
 # Install Golang
 before_install:
+  - sudo add-apt-repository ppa:gophers/archive -y
   - sudo apt-get update -qq
-  - sudo apt-get install golang -y
+  - sudo apt-get install golang-1.9-go -y
 
 # Cache gradle dependencies
 # https://docs.travis-ci.com/user/languages/android/#Caching
@@ -29,5 +38,5 @@ cache:
     - $HOME/.gradle/wrapper/
 
 script:
-  - ./gradlew clean lint
+  - ./gradlew lintVitalRelease
   - ./gradlew buildNative assembleDebug

README.md

@@ -35,7 +35,7 @@ Use `./gradlew assembleDebug` in the project directory to compile the APK.
 
 To prepare a new release, execute `./prepare-release.bash`, and follow the instructions.
 
-To check for updated gradle dependencies, run `gradle dependencyUpdates`. Additionally, the git submodule in `ext/syncthing/src/github.com/syncthing/syncthing` may need to be updated.
+To check for updated gradle dependencies, run `gradle dependencyUpdates`. Additionally, the git submodule in `go/src/github.com/syncthing/syncthing/` may need to be updated.
 
 
 ### Building on Windows

build.gradle

@@ -83,13 +83,11 @@ android {
 
 task buildNative(type: Exec) {
     outputs.upToDateWhen { false }
-    executable = './make-all.bash'
+    executable = './make-syncthing.bash'
 }
 
 task cleanNative(type: Delete) {
     delete 'src/main/jniLibs/'
-    delete 'ext/syncthing/bin/'
-    delete 'ext/syncthing/pkg/'
 }
 
 play {

ext/golang/go

@@ -1 +0,0 @@
-Subproject commit cd6b6202dd1559b3ac63179b45f1833fcfbe7eca

ext/golang/go1.4

@@ -1 +0,0 @@
-Subproject commit 50eb39bb23e8b03e823c38e844f0410d0b5325d2

ext/syncthing/src/github.com/syncthing/syncthing

@@ -1 +0,0 @@
-Subproject commit 5aade9a4a5506e5b71b2a9ab863790dca488700e

go/src/github.com/syncthing/syncthing

@@ -0,0 +1 @@
+Subproject commit cdbb32d0f043c56c6787df3f036feabc8891fd3c

make-all.bash

@@ -1,12 +0,0 @@
-#!/bin/bash -e
-
-# Build the syncthing library
-
-./make-go.bash arm
-./make-syncthing.bash arm
-
-./make-go.bash 386
-./make-syncthing.bash 386
-
-./make-go.bash arm64
-./make-syncthing.bash arm64

make-go.bash

@@ -1,96 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-export CGO_ENABLED=0
-
-if [ -z "$GOROOT_BOOTSTRAP" ]; then
-    # We need Go 1.4 to bootstrap Go 1.5
-    if [ -z $GOROOT ] || [[ $(go version) != go\ version\ go1.4* ]] ; then
-            git submodule update --init ext/golang/go1.4
-            # Build Go 1.4 for host
-            pushd ext/golang/go1.4/src
-            ./make.bash --no-clean
-            popd
-            # Add Go 1.4 to the environment
-            export GOROOT="$(pwd)/ext/golang/go1.4"
-    fi
-    # Add Go 1.4 compiler to PATH
-    export GOROOT_BOOTSTRAP=$GOROOT
-fi
-
-case "$1" in
-    arm)
-        export GOOS=linux
-        export GOARCH=arm
-        export GOARM=5
-        ;;
-    arm64)
-        export GOOS=linux
-        export GOARCH=arm64
-        unset GOARM
-        ;;
-    386)
-        export GOOS=linux
-        export GOARCH=386
-        export GO386=387
-        ;;
-    *)
-        echo "Invalid architecture"
-        exit 1
-esac
-
-unset GOPATH
-
-export GOROOT_FINAL=${MYDIR}/ext/golang/dist/go-${GOOS}-${GOARCH}
-
-if [ -d "$GOROOT_FINAL" ]; then
-    rm -r "$GOROOT_FINAL"
-fi
-mkdir -p "$GOROOT_FINAL"
-
-pushd ext/golang/go/src
-
-git reset --hard HEAD
-
-# Apply patches to Golang
-for PATCH in $MYDIR/patches/golang/all/*.patch; do
-    echo "Applying $PATCH"
-    patch -p1 <$PATCH
-done
-
-set +e
-./clean.bash
-rm -r ../bin
-rm -r ../pkg
-set -e
-
-if [ ! -e ../VERSION ]; then
-    echo "$(git describe --tags)" > ../VERSION
-fi
-
-BUILDER_EXT=bash
-case "$(uname)" in
-    *MINGW* | *WIN32* | *CYGWIN*)
-        BUILDER_EXT=bat
-        ;;
-esac
-
-./make.${BUILDER_EXT} --no-banner
-
-cp -a ../bin "${GOROOT_FINAL}"/
-cp -a ../pkg "${GOROOT_FINAL}"/
-cp -a ../src "${GOROOT_FINAL}"/
-
-if [[ -e ./make.${BUILDER_EXT} ]]; then
-    pushd ../
-    git clean -f
-    popd
-fi
-
-popd
-
-echo "Complete"
-

make-syncthing.bash

@@ -2,72 +2,63 @@
 
 set -e
 
-MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-JNIDIR="/src/main/jniLibs"
+TARGET_SDK=$(grep "targetSdkVersion" "${ROOT_DIR}/build.gradle" -m 1 | awk '{print $2}')
+# Use seperate build dir so standalone ndk isn't deleted by `gradle clean`
+BUILD_DIR="${ROOT_DIR}/gobuild"
+export GOPATH="${ROOT_DIR}/go/"
 
-case "$1" in
+cd "${ROOT_DIR}/go/src/github.com/syncthing/syncthing"
-    arm)
-        export CGO_ENABLED=0
-        export GOOS=linux
-        export GOARCH=arm
-        export GOARM=5
-        export TARGETDIR=$MYDIR$JNIDIR/armeabi
-        ;;
-    arm64)
-        export CGO_ENABLED=0
-        export GOOS=linux
-        export GOARCH=arm64
-        unset GOARM
-        export TARGETDIR=$MYDIR$JNIDIR/arm64-v8a
-        ;;
-    386)
-        export CGO_ENABLED=0
-        export GOOS=linux
-        export GOARCH=386
-        export GO386=387
-        export TARGETDIR=$MYDIR$JNIDIR/x86
-        ;;
-    *)
-        echo "Invalid architecture"
-        exit 1
-esac
-
-unset GOPATH #Set by build.go
-export GOROOT=${MYDIR}/ext/golang/dist/go-${GOOS}-${GOARCH}
-export PATH=${GOROOT}/bin:${PATH}
-
-case "$(uname)" in
-    *CYGWIN*)
-        export GOROOT=`cygpath -w $GOROOT`
-        ;;
-esac
-
-pushd ext/syncthing/src/github.com/syncthing/syncthing
-
-_GOOS=$GOOS
-unset GOOS
-_GOARCH=$GOARCH
-unset GOARCH
 
 # Make sure all tags are available for git describe
 # https://github.com/syncthing/syncthing-android/issues/872
 git fetch --tags
 
-go run build.go -goos=${_GOOS} -goarch=${_GOARCH} clean
+for ANDROID_ARCH in arm x86 arm64; do
-go run build.go -goos=${_GOOS} -goarch=${_GOARCH} -no-upgrade build
+    echo -e "Starting build for ${ANDROID_ARCH}\n"
+    case ${ANDROID_ARCH} in
+        arm)
+            GOARCH=arm
+            JNI_DIR="armeabi"
+            GCC="arm-linux-androideabi-gcc"
+            ;;
+        arm64)
+            GOARCH=arm64
+            JNI_DIR="arm64-v8a"
+            GCC="aarch64-linux-android-gcc"
+            ;;
+        x86)
+            GOARCH=386
+            JNI_DIR="x86"
+            GCC="i686-linux-android-gcc"
+            ;;
+        *)
+            echo "Invalid architecture"
+            exit 1
+    esac
 
-export GOOS=$_GOOS
+    # Build standalone NDK toolchain if it doesn't exist.
-export GOARCH=$_GOARCH
+    # https://developer.android.com/ndk/guides/standalone_toolchain.html
+    STANDALONE_NDK_DIR="${BUILD_DIR}/standalone-ndk/android-${TARGET_SDK}-${GOARCH}"
 
-mkdir -p ${TARGETDIR}
+    if [ ! -d "$STANDALONE_NDK_DIR" ]; then
-mv syncthing ${TARGETDIR}/libsyncthing.so
+        echo -e "Building standalone NDK\n"
-chmod 644 ${TARGETDIR}/libsyncthing.so
+        ${ANDROID_NDK_HOME}/build/tools/make-standalone-toolchain.sh \
+          --platform=android-${TARGET_SDK} --arch=${ANDROID_ARCH} \
+          --install-dir=${STANDALONE_NDK_DIR}
+    fi
 
-if [[ -e ./build.go ]]; then
+    echo -e "Building Syncthing\n"
-    git clean -f
+    CGO_ENABLED=1 CC="${STANDALONE_NDK_DIR}/bin/${GCC}" \
-fi
+      go run build.go -goos android -goarch ${GOARCH} -pkgdir "${BUILD_DIR}/go-packages" -no-upgrade build
 
-popd
+    # Copy compiled binary to jniLibs folder
+    TARGET_DIR="$ROOT_DIR/src/main/jniLibs/$JNI_DIR"
+    mkdir -p ${TARGET_DIR}
+    mv syncthing ${TARGET_DIR}/libsyncthing.so
 
-echo "Build Complete"
+    echo -e "Finished build for ${ANDROID_ARCH}\n"
 
+done
+
+echo -e "All builds finished"

patches/golang/all/dns.patch

@@ -1,90 +0,0 @@
---- /net/dnsconfig_unix.go	2016-08-20 08:14:05.763235602 -0400
-+++ /net/dnsconfig_unix.go	2016-08-20 08:14:05.763235602 -0400
-@@ -9,16 +9,59 @@
- package net
- 
- import (
-+	"fmt"
- 	"os"
-+	"os/exec"
-+	"strings"
- 	"sync/atomic"
- 	"time"
- )
- 
- var (
--	defaultNS   = []string{"127.0.0.1:53", "[::1]:53"}
-+	currentNS   []string
- 	getHostname = os.Hostname // variable for testing
- )
- 
-+func getDefaultNS() []string {
-+	var servers []string
-+	for _, prop := range []string{"net.dns1", "net.dns2"} {
-+		cmd := exec.Command("getprop", prop)
-+		outputBytes, err := cmd.Output()
-+		if err != nil {
-+			continue
-+		}
-+		output := strings.Trim(string(outputBytes), "\n")
-+		if ParseIP(output) != nil {
-+			servers = append(servers, JoinHostPort(output, "53"))
-+		}
-+	}
-+
-+	if len(servers) == 0 {
-+		servers = []string{"8.8.8.8:53", "8.8.4.4:53", "4.2.2.1:53"}
-+	}
-+
-+	if !slicesEqual(currentNS, servers) {
-+		fmt.Println("Using DNS servers:", servers)
-+		currentNS = servers
-+	}
-+
-+	return currentNS
-+}
-+
-+func slicesEqual(a, b []string) bool {
-+	if len(a) != len(b) {
-+		return false
-+	}
-+
-+	for i := range a {
-+		if a[i] != b[i] {
-+			return false
-+		}
-+	}
-+
-+	return true
-+}
-+
- type dnsConfig struct {
- 	servers    []string      // server addresses (in host:port form) to use
- 	search     []string      // rooted suffixes to append to local name
-@@ -42,7 +85,7 @@ func dnsReadConfig(filename string) *dnsConfig {
- 	}
- 	file, err := open(filename)
- 	if err != nil {
--		conf.servers = defaultNS
-+		conf.servers = getDefaultNS()
- 		conf.search = dnsDefaultSearch()
- 		conf.err = err
- 		return conf
-@@ -51,7 +94,7 @@ func dnsReadConfig(filename string) *dnsConfig {
- 	if fi, err := file.file.Stat(); err == nil {
- 		conf.mtime = fi.ModTime()
- 	} else {
--		conf.servers = defaultNS
-+		conf.servers = getDefaultNS()
- 		conf.search = dnsDefaultSearch()
- 		conf.err = err
- 		return conf
-@@ -130,7 +173,7 @@ func dnsReadConfig(filename string) *dnsConfig {
- 		}
- 	}
- 	if len(conf.servers) == 0 {
--		conf.servers = defaultNS
-+		conf.servers = getDefaultNS()
- 	}
- 	if len(conf.search) == 0 {
- 		conf.search = dnsDefaultSearch()

prepare-release.bash

@@ -16,7 +16,7 @@ Checking for Syncthing Update
 -----------------------------
 "
 PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd "ext/syncthing/src/github.com/syncthing/syncthing/"
+cd "go/src/github.com/syncthing/syncthing/"
 git fetch
 CURRENT_TAG=$(git describe)
 LATEST_TAG=$(git tag --sort=taggerdate | awk '!/rc/' | tail -1)
@@ -24,7 +24,7 @@ if [ ${CURRENT_TAG} != ${LATEST_TAG} ]
 then
     git checkout -f ${LATEST_TAG}
     cd ${PROJECT_DIR}
-    git add "ext/syncthing/src/github.com/syncthing/syncthing"
+    git add "go/src/github.com/syncthing/syncthing/"
     git commit -m "Updated Syncthing to $LATEST_TAG"
     ./gradlew cleanNative buildNative
 fi