Add GameType filter, and add ForgeAPI Verify Files to validate Mod Downloads (#1337)

Co-authored-by: christopher blodgett <christopher.blodgett@gmail.com>

DEVELOPMENT.md

@@ -15,21 +15,25 @@ Individual scripts can be iteratively developed, debugged, and tested using the
 
 First, build a baseline of the image to include the packages needed by existing or new scripts:
 
-PowerShell:
+PowerShell: (Example of building and testing ForgeAPI)
 ```powershell
+$env:MODS_FORGEAPI_KEY='$2a$...'
+$env:FOLDER_TO_TEST="forgeapimods_projectids"
 $env:IMAGE_TO_TEST="mc-dev"
 docker build -t $env:IMAGE_TO_TEST .
-pushd tests/setuponlytests/vanillatweaks_file/
+pushd "tests/setuponlytests/$env:FOLDER_TO_TEST/"
 docker-compose run mc
 docker-compose down --remove-orphans
 popd
 ```
 
-Bash:
+Bash: (Example of building and testing ForgeAPI)
 ```bash
-export IMAGE_TO_TEST=mc-dev
+export MODS_FORGEAPI_KEY='$2a$...'
+export FOLDER_TO_TEST="forgeapimods_file"
+export IMAGE_TO_TEST="mc-dev"
 docker build -t $IMAGE_TO_TEST .
-pushd tests/setuponlytests/vanillatweaks_file/
+pushd tests/setuponlytests/$FOLDER_TO_TEST/
 docker-compose run mc
 docker-compose down --remove-orphans
 popd

Dockerfile

@@ -61,7 +61,7 @@ RUN easy-add --var os=${TARGETOS} --var arch=${TARGETARCH}${TARGETVARIANT} \
   --var version=0.1.1 --var app=maven-metadata-release --file {{.app}} \
   --from https://github.com/itzg/{{.app}}/releases/download/{{.version}}/{{.app}}_{{.version}}_{{.os}}_{{.arch}}.tar.gz
 
-ARG MC_HELPER_VERSION=1.16.5
+ARG MC_HELPER_VERSION=1.16.6
 ARG MC_HELPER_BASE_URL=https://github.com/itzg/mc-image-helper/releases/download/v${MC_HELPER_VERSION}
 RUN curl -fsSL ${MC_HELPER_BASE_URL}/mc-image-helper-${MC_HELPER_VERSION}.tgz \
   | tar -C /usr/share -zxf - \

README.md

@@ -753,6 +753,11 @@ You may also download or copy over individual mods using the `MODS` environment
 
 This is more complicated because you will be pulling/using the latest mod for the release of your game. To get started make sure you have a [CursedForge API Key](https://docs.curseforge.com/#getting-started). Then use the environmental parameters in your docker build.
 
+Please be aware of the following when using these options for your mods:
+* Mod Release types: Release, Beta, and Alpha.
+* Mod dependencies: Required and Optional
+* Mod family: Fabric, Forge, and Bukkit.
+
 Parameters to use the ForgeAPI:
 
 * `MODS_FORGEAPI_KEY` - Required
@@ -760,6 +765,7 @@ Parameters to use the ForgeAPI:
 * `MODS_FORGEAPI_PROJECTIDS` - Required or use MODS_FORGEAPI_FILE
 * `MODS_FORGEAPI_RELEASES` - Default is release, Options: [Release|Beta|Alpha]
 * `MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES` - Default is False, attempts to download required mods (releaseType Release) defined in Forge.
+* `MODS_FORGEAPI_IGNORE_GAMETYPE` - Default is False, Allows for filtering mods on family type: FORGE, FABRIC, and BUKKIT. (Does not filter for Vanilla or custom)
 * `REMOVE_OLD_FORGEAPI_MODS` - Default is False
 * `REMOVE_OLD_DATAPACKS_DEPTH` - Default is 1
 * `REMOVE_OLD_DATAPACKS_INCLUDE` - Default is *.jar

scripts/start-deployCustom

@@ -31,6 +31,7 @@ else
 
 fi
 
-export FAMILY=HYBRID
+# Allow for overriding Family on custom for testing.
+export FAMILY="${FAMILY:-HYBRID}"
 
 exec ${SCRIPTS:-/}start-setupWorld $@

scripts/start-setupForgeApiMods

@@ -8,6 +8,7 @@ set -e -o pipefail
 : "${MODS_FORGEAPI_FILE:=}"
 : "${MODS_FORGEAPI_RELEASES:=RELEASE}"
 : "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES:=false}"
+: "${MODS_FORGEAPI_IGNORE_GAMETYPE:=false}"
 : "${REMOVE_OLD_MODS_DEPTH:=1} "
 : "${REMOVE_OLD_MODS_INCLUDE:=*.jar}"
 
@@ -15,6 +16,7 @@ set -e -o pipefail
 FORGEAPI_BASE_URL=${FORGEAPI_BASE_URL:-https://api.curseforge.com/v1}
 RELEASE_NUMBER_FILTER=1
 MINECRAFT_GAME_ID=432
+FILTER_BY_FAMILY=false
 out_dir=/data/mods
 
 # shellcheck source=start-utils
@@ -22,10 +24,17 @@ out_dir=/data/mods
 isDebugging && set -x
 
 # Remove old mods/plugins
-if isTrue "${REMOVE_OLD_FORGEAPI_MODS}" && [ -z "${MODS_FORGEAPI_FILE}" ]; then
+if isTrue "${REMOVE_OLD_FORGEAPI_MODS}" && [ -z "${MODS_FORGEAPI_KEY}" ]; then
   removeOldMods /data/mods
 fi
 
+# Family filter is on by default for Forge, Fabric, and Bukkit
+updateFamilyFilter(){
+  if isFamily "FORGE" "FABRIC" "BUKKIT"; then
+    FILTER_BY_FAMILY=true
+  fi
+}
+
 ensureModKey(){
   if [ -z "$MODS_FORGEAPI_KEY" ]; then
     log "ERROR: MODS_FORGEAPI_KEY REQUIRED to Connect to FORGE API, you supplied: ${MODS_FORGEAPI_KEY}"
@@ -97,18 +106,27 @@ modFileByProjectID(){
     # Checking for a individual release type input, if not use global
     if [ $project_id_release_type ]; then
       updateReleaseNumber $project_id_release_type
+      unset project_id_release_type 
     else
       updateReleaseNumber $MODS_FORGEAPI_RELEASES
     fi
+
     # grabs the highest ID of the releaseTypes selected.
     # Default is 1 for Release, Beta is 2, and Alpha is 3. Using less than we can validate highest release.
     if [ $project_id_file_name ]; then
+      # Looks for file by name
       current_project_file=$(jq -n "$project_files" | jq --arg FILE_NAME "$project_id_file_name" -jc '
-        .data | map(select(.fileName<=($FILE_NAME))) | .[0]')
+        .data | map(select(.fileName<=($FILE_NAME))) | .[0] // empty')
+    elif $( ! isTrue "$MODS_FORGEAPI_IGNORE_GAMETYPE" ) && $FILTER_BY_FAMILY ; then
+      # Looks for file by version and server type in lowercase
+      current_project_file=$(jq -n "$project_files" | jq --arg RELEASE_FILTER "$RELEASE_NUMBER_FILTER" --arg GAME_TYPE ${FAMILY,,} -jc '
+        .data | sort_by(.id) | reverse | map(select(.gameVersions[] | ascii_downcase | contains ($GAME_TYPE))) | map(select(.releaseType<=($RELEASE_FILTER|tonumber))) | .[0] // empty')
     else
+      # Looks for file by version only.
       current_project_file=$(jq -n "$project_files" | jq --arg RELEASE_FILTER "$RELEASE_NUMBER_FILTER" -jc '
-        .data | sort_by(.id) | reverse | map(select(.releaseType<=($RELEASE_FILTER|tonumber))) | .[0]')
+        .data | sort_by(.id) | reverse | map(select(.releaseType<=($RELEASE_FILTER|tonumber))) | .[0] // empty')
     fi
+
     # Logic to grab the latest release over the entire pagination
     if [ ! "$PROJECT_FILE" ]; then
       PROJECT_FILE=$current_project_file
@@ -128,6 +146,10 @@ modFileByProjectID(){
     # Increment start index to new set.
     index=$(($index + $pageSize))
   done
+  if [ ! "$PROJECT_FILE" ]; then
+    log "ERROR: Unable to retrieve any files for ${project_id}, Release Type: ${RELEASE_NUMBER_FILTER}, FAMILY_TYPE: ${FAMILY,,}"
+    exit 2
+  fi
 }
 
 downloadModPackfromModFile() {
@@ -143,7 +165,7 @@ downloadModPackfromModFile() {
   # trys to make the output directory incase it doesnt exist.
   mkdir -p "$out_dir"
   echo "Downloading ${download_url}"
-  if ! get -o "${out_dir}/${file_name}" $download_url ; then 
+  if ! get --skip-up-to-date -o "${out_dir}/${file_name}" $download_url ; then
     log "ERROR: failed to download from ${download_url}"
     exit 2
   fi
@@ -186,17 +208,18 @@ downloadDependencies(){
 # Use forge api json file to filter and download the correct mods
 if [ "$MODS_FORGEAPI_FILE" ] && [ -z "$MODS_FORGEAPI_PROJECTIDS" ]; then
   ensureModKey
+  updateFamilyFilter
   if [ ! -f "$MODS_FORGEAPI_FILE" ]; then
     log "ERROR: given MODS_FORGEAPI_FILE file does not exist"
     exit 2
   fi
 
   # Needs loop here to look up release types befor calling download.
-  jq -n "$required_dependencies" | jq -c '.[]?' | while read current_project; do
+  jq -c '.[]?' $MODS_FORGEAPI_FILE | while read current_project; do
     # Per stack overflow we can use //empty to return empty string that works with -z
-    project_id=$(jq -n "$current_project" | jq -jc '.projectId // empty' )
+    project_id=$(jq -n "$current_project" | jq -r '.projectId // empty' )
-    current_release_type=$(jq -n "$current_project" | jq -jc '.releaseType // empty' )
+    current_release_type=$(jq -n "$current_project" | jq -r '.releaseType // empty' )
-    current_file_name=$(jq -n "$current_project" | jq -jc '.fileName // empty' )
+    current_file_name=$(jq -n "$current_project" | jq -r '.fileName // empty' )
 
     modFileByProjectID $project_id $current_release_type $current_file_name
     downloadModPackfromModFile
@@ -209,6 +232,7 @@ fi
 # Use only project ids and global release data.
 if [ "$MODS_FORGEAPI_PROJECTIDS" ] && [ -z "$MODS_FORGEAPI_FILE" ]; then
   ensureModKey
+  updateFamilyFilter
   for project_id in ${MODS_FORGEAPI_PROJECTIDS//,/ }; do
     modFileByProjectID $project_id 
     downloadModPackfromModFile

scripts/start-setupWorld

@@ -34,6 +34,7 @@ if [[ "$WORLD" ]] && ( isTrue "${FORCE_WORLD_COPY}" || [ ! -d "$worldDest" ] );
     # Stage contents so that the correct subdirectory can be picked off
     mkdir -p /tmp/world-data
     if ! extract "$WORLD" /tmp/world-data; then
+      log "ERROR extracting world from $WORLD"
       exit 1
     fi
 

scripts/start-utils

@@ -190,13 +190,13 @@ function get() {
 
 function isFamily() {
   for f in "${@}"; do
-    if [[ $FAMILY == "$f" ]]; then
+    if [[ ${FAMILY^^} == "${f^^}" ]]; then
       return 0
     fi
   done
-
   return 1
 }
+
 function isType() {
   for t in "${@}"; do
     # shellcheck disable=SC2153

tests/setuponlytests/forgeapimods_file/docker-compose.yml

@@ -7,12 +7,20 @@ services:
     environment:
       EULA: "TRUE"
       SETUP_ONLY: "TRUE"
+      # Using custom to bypass Fabric setup
+      TYPE: CUSTOM
+      # Using family to test FORGEAPI Family filter.
+      FAMILY: FABRIC
+      CUSTOM_SERVER: /servers/fake.jar
       VERSION: ${MINECRAFT_VERSION:-LATEST}
       MODS_FORGEAPI_FILE: /config/forgeapi_mods.json
       # Key is defined in .github/workflows/pr.yml and ci.yml
       # This should be coming from github secrets.
       MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}
       REMOVE_OLD_FORGEAPI_MODS: "TRUE"
-      MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES: "FALSE"
+      # Validates that Fabric API gets download as a dependency.
+      MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES: "TRUE"
     volumes:
       - ./forgeapi_mods.json:/config/forgeapi_mods.json:ro
+      - ./data:/data
+      - ./fake.jar:/servers/fake.jar

tests/setuponlytests/forgeapimods_file/forgeapi_mods.json

@@ -1,17 +1,11 @@
 [{
-        "name": "fabric api",
+        "name": "On A Stick [FABRIC]",
-        "projectId": "306612",
+        "projectId": "550544",
         "releaseType": "release"
     },
     {
         "name": "Fabric Voice Mod",
         "projectId": "416089",
         "releaseType": "beta"
-    },
-    {
-        "name": "Biomes o plenty",
-        "projectId": "220318",
-        "fileName": "BiomesOPlenty-1.18.1-15.0.0.100-universal.jar",
-        "releaseType": "release"
     }
-]
+]

tests/setuponlytests/forgeapimods_file/verify.sh

@@ -0,0 +1,5 @@
+# Validates specific beta call out for specific mod:
+mc-image-helper assert fileExists "/data/mods/voicechat-fabric*"
+mc-image-helper assert fileExists "/data/mods/onastick-fabric*"
+# Dependent of on a stick:
+mc-image-helper assert fileExists "/data/mods/fabric-api*"

tests/setuponlytests/forgeapimods_gametype/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3"
+
+services:
+  mc:
+    restart: "no"
+    image: ${IMAGE_TO_TEST:-itzg/minecraft-server}
+    environment:
+      EULA: "TRUE"
+      SETUP_ONLY: "TRUE"
+      # Using custom to bypass Fabric setup
+      TYPE: CUSTOM
+      # Using family to test FORGEAPI Family filter.
+      FAMILY: FABRIC
+      CUSTOM_SERVER: /servers/fake.jar
+      VERSION: ${MINECRAFT_VERSION:-LATEST}
+      # Validate Skip Gametype Filter:
+      MODS_FORGEAPI_IGNORE_GAMETYPE: "TRUE"
+      # Validates that Biomes does not download terrablender
+      # Using default false for testing: 
+      # MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES: "FALSE"
+      # Contains mix of Forge and Fabric mods
+      MODS_FORGEAPI_PROJECTIDS: 306612,416089,220318
+      # Allows for Beta releases of 416089 the Fabric Voice Mod 
+      MODS_FORGEAPI_RELEASES: BETA
+      MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}
+      REMOVE_OLD_FORGEAPI_MODS: "FALSE"
+    volumes:
+      - ./data:/data
+      - ./fake.jar:/servers/fake.jar

tests/setuponlytests/forgeapimods_gametype/require.sh

@@ -0,0 +1 @@
+[[ $MODS_FORGEAPI_KEY ]] || exit 1

tests/setuponlytests/forgeapimods_gametype/verify.sh

@@ -0,0 +1,5 @@
+mc-image-helper assert fileExists "/data/mods/BiomesOPlenty*"
+# testing dependencies don't get downloaded when download dependencies is set to false.
+! mc-image-helper assert fileExists "/data/mods/TerraBlender*"
+mc-image-helper assert fileExists "/data/mods/voicechat-fabric*"
+mc-image-helper assert fileExists "/data/mods/fabric-api*"

tests/setuponlytests/forgeapimods_projectids/docker-compose.yml

@@ -7,10 +7,20 @@ services:
     environment:
       EULA: "TRUE"
       SETUP_ONLY: "TRUE"
+      # Using custom to bypass Fabric setup
+      TYPE: CUSTOM
+      # Validate Skip Gametype Filter for vanilla 
+      #  - Currently we do not support filtering on vanilla.
+      FAMILY: VANILLA
+      CUSTOM_SERVER: /servers/fake.jar
       VERSION: ${MINECRAFT_VERSION:-LATEST}
       MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES: "TRUE"
+      # Contains mix of Forge and Fabric mods
       MODS_FORGEAPI_PROJECTIDS: 306612,416089,220318
       # Allows for Beta releases of 416089 the Fabric Voice Mod 
       MODS_FORGEAPI_RELEASES: BETA
       MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}
       REMOVE_OLD_FORGEAPI_MODS: "TRUE"
+    volumes:
+      - ./data:/data
+      - ./fake.jar:/servers/fake.jar

tests/setuponlytests/forgeapimods_projectids/verify.sh

@@ -0,0 +1,4 @@
+mc-image-helper assert fileExists "/data/mods/BiomesOPlenty*"
+mc-image-helper assert fileExists "/data/mods/TerraBlender*"
+mc-image-helper assert fileExists "/data/mods/voicechat-fabric*"
+mc-image-helper assert fileExists "/data/mods/fabric-api*"