Fall 2024 recompile of the natives (#1416)

This time around, instead of the very ad hoc approach I have historically taken, this time we actually have a convenient script that takes care of basically all the hard work necessary, including cross-compilation.

We compile natives for Ubuntu 20.04 LTS (OpenSSL 1.1.x native and libdeflate) and Ubuntu 22.04 LTS (OpenSSL 3.x.x native), for both x86_64 and aarch64.

The macOS natives have also been recompiled on macOS Sonoma.

native/build-support/build-all-linux-natives.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -e
+
+# make sure we're in the correct directory - the top-level `native` directory
+cd "$(dirname "$0")/.." || exit 1
+
+ARCHS=(x86_64 aarch64)
+BASE_DOCKERFILE_VARIANTS=(ubuntu-focal ubuntu-jammy)
+
+for variant in "${BASE_DOCKERFILE_VARIANTS[@]}"; do
+  docker_platforms=""
+  for arch in "${ARCHS[@]}"; do
+    docker_platforms="$docker_platforms --platform linux/${arch}"
+  done
+
+  echo "Building base build image for $variant..."
+  docker build -t velocity-native-build:$variant $docker_platforms -f build-support/$variant.Dockerfile .
+done
+
+for arch in "${ARCHS[@]}"; do
+  for variant in "${BASE_DOCKERFILE_VARIANTS[@]}"; do
+    echo "Building native crypto for $arch on $variant..."
+
+    docker run --rm -v "$(pwd)":/app --platform linux/${arch} velocity-native-build:$variant /bin/bash -c "cd /app && ./build-support/compile-linux-crypto.sh"
+  done
+
+  # Use only the oldest variant for the compression library
+  variant=${BASE_DOCKERFILE_VARIANTS[0]}
+  echo "Building native compression for $arch on $variant..."
+  docker run --rm -v "$(pwd)":/app --platform linux/${arch} velocity-native-build:$variant /bin/bash -c "cd /app && ./build-support/compile-linux-compress.sh"
+done

native/build-support/compile-linux-compress.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ ! "$CC" ]; then
+  # The libdeflate authors recommend that we build using GCC as it produces "slightly faster binaries":
+  # https://github.com/ebiggers/libdeflate#for-unix
+  export CC=gcc
+fi
+
+if [ ! -d libdeflate ]; then
+  echo "Cloning libdeflate..."
+  git clone --branch v1.21 --single-branch https://github.com/ebiggers/libdeflate.git
+fi
+
+echo "Compiling libdeflate..."
+cd libdeflate || exit
+rm -rf build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -B build && cmake --build build --target libdeflate_static
+cd ..
+
+CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack -Wall -Werror -fomit-frame-pointer"
+ARCH=$(uname -m)
+mkdir -p src/main/resources/linux_$ARCH
+$CC $CFLAGS -Ilibdeflate src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \
+    libdeflate/build/libdeflate.a -o src/main/resources/linux_$ARCH/velocity-compress.so

native/build-support/compile-linux-crypto.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+openssl_version=$(openssl version | awk '{print $2}')
+
+# Extract the major and minor version numbers
+major_version=$(echo "$openssl_version" | cut -d. -f1)
+minor_version=$(echo "$openssl_version" | cut -d. -f2)
+
+# Determine the appropriate file name based on the version
+if [ "$major_version" -eq 1 ] && [ "$minor_version" -eq 1 ]; then
+    filename="velocity-cipher-ossl11x.so"
+elif [ "$major_version" -eq 3 ]; then
+    filename="velocity-cipher-ossl30x.so"
+else
+    echo "Unsupported OpenSSL version: $openssl_version"
+    exit 1
+fi
+
+if [ ! "$CC" ]; then
+  export CC=gcc
+fi
+
+output_file="velocity-cipher.so"
+if [ -n "$OPENSSL_VERSION" ]; then
+  output_file="velocity-cipher-ossl${OPENSSL_VERSION}.so"
+fi
+
+CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack -Wall -Werror -fomit-frame-pointer"
+ARCH=$(uname -m)
+mkdir -p src/main/resources/linux_$ARCH
+$CC $CFLAGS -shared src/main/c/jni_util.c src/main/c/jni_cipher_openssl.c \
+    -o src/main/resources/linux_$ARCH/$filename -lcrypto

native/build-support/compile-macos.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ ! "$CC" ]; then
+  export CC=clang
+fi
+
+if [ ! -d libdeflate ]; then
+  echo "Cloning libdeflate..."
+  git clone --branch v1.21 --single-branch https://github.com/ebiggers/libdeflate.git
+fi
+
+echo "Compiling libdeflate..."
+cd libdeflate || exit
+cmake -B build && cmake --build build --target libdeflate_static
+cd ..
+
+CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin/ -fPIC -shared -Wall -Werror -fomit-frame-pointer"
+ARCH=$(uname -m)
+mkdir -p src/main/resources/macos_$ARCH
+$CC $CFLAGS -Ilibdeflate src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \
+    libdeflate/build/libdeflate.a -o src/main/resources/macos_$ARCH/velocity-compress.dylib
+$CC $CFLAGS -shared src/main/c/jni_util.c src/main/c/jni_cipher_macos.c \
+    -o src/main/resources/macos_$ARCH/velocity-cipher.dylib -lSystem

native/build-support/ubuntu-focal.Dockerfile

@@ -0,0 +1,20 @@
+# Use the official Eclipse Temurin 17.0.12_7-jdk-focal image as the base image.
+# We compile for Ubuntu Focal Fossa (20.04 LTS) as it is still supported until 2025, and the crypto
+# native is specific to a given OpenSSL version.
+FROM eclipse-temurin:17.0.12_7-jdk-focal
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    curl \
+    git \
+    unzip \
+    build-essential \
+    cmake \
+    openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create a non-root user
+RUN useradd -m -s /bin/bash -u 1000 -U user
+USER user

native/build-support/ubuntu-jammy.Dockerfile

@@ -0,0 +1,19 @@
+# Use the official Eclipse Temurin 17.0.12_7-jdk-jammy image as the base image.
+# We compile for Ubuntu Jammy Jellyfish (22.04 LTS) as it supports OpenSSL 3.0.
+FROM eclipse-temurin:17.0.12_7-jdk-jammy
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    curl \
+    git \
+    unzip \
+    build-essential \
+    cmake \
+    openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create a non-root user
+RUN useradd -m -s /bin/bash -u 1000 -U user
+USER user