Make gcc-aarch64 reproducible (#366)

This fixes https://github.com/postmarketOS/binary-package-repo/issues/1

GCC generates hardlinks between files `A` and `B` in its `make install` step. The problem is, that `tar` randomly packages `A` as full binary, and links `B` to `A`, or the other way around! I was able to reproduce this issue consistently when re-building `gcc-aarch64` on Travis CI (interestingly, this did not appear for `gcc-armhf`).

The fix is, to delete `B` and create a symlink `B` that points to `A` instead.

aports/cross/gcc-aarch64/APKBUILD

@@ -12,6 +12,41 @@ LANG_FORTRAN=false
 LANG_ADA=false
 options="!strip !tracedeps"
 
+# Wrap the package function, to make the resulting package
+# lazy-reproducible
+package() {
+    # Repack the *.a files to be reproducible (see #64)
+    _temp="$_builddir"/_reproducible-patch
+    cd "$_builddir"
+    for f in $(find -name '*.a'); do
+        # Copy to a temporary folder
+        echo "Repack $f to be reproducible"
+        mkdir -p "$_temp"
+        cd "$_temp"
+        cp "$_builddir"/"$f" .
+
+        # Repack with a sorted file order
+        ar x *.a
+        rm *.a
+        ar r sorted.a $(find -name '*.o' | sort)
+
+        # Copy back and clean up
+        cp -v sorted.a "$_builddir"/"$f"
+        cd ..
+        rm -r "$_temp"
+    done
+
+    # Unmodified package function from the gcc APKBUILD
+    _package
+
+    # Workaround for: postmarketOS/binary-package-repo#1
+    echo "Replacing hardlinks with symlinks"
+    rm -v "$pkgdir"/usr/bin/"$CTARGET"-c++
+    ln -s -v /usr/bin/"$CTARGET"-g++ "$pkgdir"/usr/bin/"$CTARGET"-c++
+    rm -v "$pkgdir"/usr/bin/"$CTARGET"-gcc-"$pkgver"
+    ln -s -v /usr/bin/"$CTARGET"-gcc "$pkgdir"/usr/bin/"$CTARGET"-gcc-"$pkgver"
+}
+
 pkgname="gcc-aarch64"
 pkgver=6.4.0
 [ "$BOOTSTRAP" = "nolibc" ] && pkgname="gcc-pass2"
@@ -320,27 +355,7 @@ build() {
 	make
 }
 
-package() {
+_package() {
-            # Repack the *.a files to be reproducible (see #64)
-            _temp="$_builddir"/_reproducible-patch
-            cd "$_builddir"
-            for f in $(find -name '*.a'); do
-                # Copy to a temporary folder
-                echo "Repack $f to be reproducible"
-                mkdir -p "$_temp"
-                cd "$_temp"
-                cp "$_builddir"/"$f" .
-
-                # Repack with a sorted file order
-                ar x *.a
-                rm *.a
-                ar r sorted.a $(find -name '*.o' | sort)
-
-                # Copy back and clean up
-                cp -v sorted.a "$_builddir"/"$f"
-                cd ..
-                rm -r "$_temp"
-            done
 	cd "$_builddir"
 	make -j1 DESTDIR="${pkgdir}" install
 

aports/cross/gcc-armhf/APKBUILD

@@ -12,6 +12,41 @@ LANG_FORTRAN=false
 LANG_ADA=false
 options="!strip !tracedeps"
 
+# Wrap the package function, to make the resulting package
+# lazy-reproducible
+package() {
+    # Repack the *.a files to be reproducible (see #64)
+    _temp="$_builddir"/_reproducible-patch
+    cd "$_builddir"
+    for f in $(find -name '*.a'); do
+        # Copy to a temporary folder
+        echo "Repack $f to be reproducible"
+        mkdir -p "$_temp"
+        cd "$_temp"
+        cp "$_builddir"/"$f" .
+
+        # Repack with a sorted file order
+        ar x *.a
+        rm *.a
+        ar r sorted.a $(find -name '*.o' | sort)
+
+        # Copy back and clean up
+        cp -v sorted.a "$_builddir"/"$f"
+        cd ..
+        rm -r "$_temp"
+    done
+
+    # Unmodified package function from the gcc APKBUILD
+    _package
+
+    # Workaround for: postmarketOS/binary-package-repo#1
+    echo "Replacing hardlinks with symlinks"
+    rm -v "$pkgdir"/usr/bin/"$CTARGET"-c++
+    ln -s -v /usr/bin/"$CTARGET"-g++ "$pkgdir"/usr/bin/"$CTARGET"-c++
+    rm -v "$pkgdir"/usr/bin/"$CTARGET"-gcc-"$pkgver"
+    ln -s -v /usr/bin/"$CTARGET"-gcc "$pkgdir"/usr/bin/"$CTARGET"-gcc-"$pkgver"
+}
+
 pkgname="gcc-armhf"
 pkgver=6.4.0
 [ "$BOOTSTRAP" = "nolibc" ] && pkgname="gcc-pass2"
@@ -320,27 +355,7 @@ build() {
 	make
 }
 
-package() {
+_package() {
-            # Repack the *.a files to be reproducible (see #64)
-            _temp="$_builddir"/_reproducible-patch
-            cd "$_builddir"
-            for f in $(find -name '*.a'); do
-                # Copy to a temporary folder
-                echo "Repack $f to be reproducible"
-                mkdir -p "$_temp"
-                cd "$_temp"
-                cp "$_builddir"/"$f" .
-
-                # Repack with a sorted file order
-                ar x *.a
-                rm *.a
-                ar r sorted.a $(find -name '*.o' | sort)
-
-                # Copy back and clean up
-                cp -v sorted.a "$_builddir"/"$f"
-                cd ..
-                rm -r "$_temp"
-            done
 	cd "$_builddir"
 	make -j1 DESTDIR="${pkgdir}" install
 

pmb/aportgen/core.py

@@ -62,7 +62,9 @@ def rewrite(args, pkgname, path_original, fields={}, replace_pkgname=None,
         "\n",
     ]
     for line in below_header.split("\n"):
-        lines_new += line.strip() + "\n"
+        if not line[:8].strip():
+            line = line[8:]
+        lines_new += line.rstrip() + "\n"
 
     # Copy/modify lines, skip Maintainer/Contributor
     path = args.work + "/aportgen/APKBUILD"

pmb/aportgen/gcc.py

@@ -51,19 +51,10 @@ def generate(args, pkgname):
         LANG_FORTRAN=false
         LANG_ADA=false
         options="!strip !tracedeps"
-    """
 
-    replace_simple = {
+        # Wrap the package function, to make the resulting package
-        # Do not package libstdc++, do not add "g++-$ARCH" here (already
+        # lazy-reproducible
-        # did that explicitly in the subpackages variable above, so
+        package() {
-        # pmbootstrap picks it up properly).
-        '*subpackages="$subpackages libstdc++:libcxx:*': None,
-
-        # libstdc++.a is not reproducible by default (.a files are archives of
-        # object files, and these object files are inside the .a file in a random
-        # order!). The best way would be to patch this upstream in gcc, but for now
-        # we repackage the .a files to make sure, that they are reproducible.
-        '*package() {*': """package() {
             # Repack the *.a files to be reproducible (see #64)
             _temp="$_builddir"/_reproducible-patch
             cd "$_builddir"
@@ -83,7 +74,28 @@ def generate(args, pkgname):
                 cp -v sorted.a "$_builddir"/"$f"
                 cd ..
                 rm -r "$_temp"
-            done"""
+            done
+
+            # Unmodified package function from the gcc APKBUILD
+            _package
+
+            # Workaround for: postmarketOS/binary-package-repo#1
+            echo "Replacing hardlinks with symlinks"
+            rm -v "$pkgdir"/usr/bin/"$CTARGET"-c++
+            ln -s -v /usr/bin/"$CTARGET"-g++ "$pkgdir"/usr/bin/"$CTARGET"-c++
+            rm -v "$pkgdir"/usr/bin/"$CTARGET"-gcc-"$pkgver"
+            ln -s -v /usr/bin/"$CTARGET"-gcc "$pkgdir"/usr/bin/"$CTARGET"-gcc-"$pkgver"
+        }
+    """
+
+    replace_simple = {
+        # Do not package libstdc++, do not add "g++-$ARCH" here (already
+        # did that explicitly in the subpackages variable above, so
+        # pmbootstrap picks it up properly).
+        '*subpackages="$subpackages libstdc++:libcxx:*': None,
+
+        # Rename package to _package, so we can wrap it (see above)
+        '*package() {*': "_package() {"
     }
 
     pmb.aportgen.core.rewrite(

pmb/challenge/apk_file.py

@@ -100,6 +100,10 @@ def apk(args, apk_a, apk_b, stop_after_first_error=False):
                     member_a = tar_a.getmember(name)
                     member_b = tar_b.getmember(name)
                     if member_a.type != member_b.type:
+                        logging.info("NOTE: " + name + " in " + apk_a + ":")
+                        tar_a.list(members=[member_a])
+                        logging.info("NOTE: " + name + " in " + apk_b + ":")
+                        tar_b.list(members=[member_b])
                         raise RuntimeError(
                             "Entry '" + name + "' has a different type!")