From 14a1e5fb2afe9c372646ce26a9b2b92c517067b3 Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Sun, 1 Mar 2015 21:39:38 -0700 Subject: [PATCH 1/7] Add initial version of forge download scraper --- minecraft-server/minecraftforge_spider.py | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 minecraft-server/minecraftforge_spider.py diff --git a/minecraft-server/minecraftforge_spider.py b/minecraft-server/minecraftforge_spider.py new file mode 100644 index 00000000..799d0fb2 --- /dev/null +++ b/minecraft-server/minecraftforge_spider.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import scrapy +from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.selector import Selector + +import re + +class Forge(scrapy.Item): + versions = scrapy.Field() + latest = scrapy.Field() + +class ForgeVersions(scrapy.Item): + id = scrapy.Field() + minecraft = scrapy.Field() + type = scrapy.Field() + time = scrapy.Field() + url = scrapy.Field() + +class ForgeLatest(scrapy.Item): + forge_latest = scrapy.Field() + forge_recommended = scrapy.Field() + +class ForgeSpider(CrawlSpider): + name = "ForgeSpider" + allowed_domains = ["minecraftforge.net"] + start_urls = ['http://files.minecraftforge.net'] + + def parse(self, response): + forge = Forge() + forge['versions'] = [] + forge['latest'] = ForgeLatest() + + selector = Selector(response) + rows = selector.xpath('//table[@id="promotions_table"]//tr') + header = rows.pop(0) + for row in rows: + cells = row.xpath('td') + + id = cells[1].xpath('text()').extract() + minecraft = cells[2].xpath('text()').extract() + type = cells[0].xpath('text()') + time = cells[3].xpath('text()') + url = cells[4].xpath('a[text()="Installer"]/@href') + + #if has version + has_version = re.match('(.+)\-.+', ''.join(type.extract())) + if has_version: + download = ForgeVersions() + download['id'] = id + download['minecraft'] = minecraft + download['type'] = 'forge_' + ''.join(type.re('([a-zA-Z]+)')).lower() + download['time'] = time.extract() + download['url'] = url.re('http://adf.ly/\d+/(.+)') + + forge['versions'].append(download) + else: + is_recommended = re.match('Recommended', ''.join(type.extract())) + if is_recommended: + download = ForgeLatest() + forge['latest']['forge_recommended'] = id + else: + download = ForgeLatest() + forge['latest']['forge_latest'] = id + + + return forge + From 87140179f272fa66af254c9e9c2512cc22b4b0fa Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Wed, 4 Mar 2015 16:55:57 -0700 Subject: [PATCH 2/7] Add option for Vanilla or Forge server --- minecraft-server/Dockerfile | 1 + minecraft-server/minecraftforge_spider.py | 68 ----------------------- minecraft-server/start-minecraft.sh | 59 ++++++++++++++------ 3 files changed, 44 insertions(+), 84 deletions(-) delete mode 100644 minecraft-server/minecraftforge_spider.py diff --git a/minecraft-server/Dockerfile b/minecraft-server/Dockerfile index ef686025..84b9accb 100644 --- a/minecraft-server/Dockerfile +++ b/minecraft-server/Dockerfile @@ -26,4 +26,5 @@ ENV UID 1000 ENV MOTD A Minecraft Server Powered by Docker ENV LEVEL world ENV JVM_OPTS -Xmx1024M -Xms1024M +ENV TYPE VANILLA ENV VERSION LATEST diff --git a/minecraft-server/minecraftforge_spider.py b/minecraft-server/minecraftforge_spider.py deleted file mode 100644 index 799d0fb2..00000000 --- a/minecraft-server/minecraftforge_spider.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -import scrapy -from scrapy.contrib.spiders import CrawlSpider, Rule -from scrapy.contrib.linkextractors import LinkExtractor -from scrapy.selector import Selector - -import re - -class Forge(scrapy.Item): - versions = scrapy.Field() - latest = scrapy.Field() - -class ForgeVersions(scrapy.Item): - id = scrapy.Field() - minecraft = scrapy.Field() - type = scrapy.Field() - time = scrapy.Field() - url = scrapy.Field() - -class ForgeLatest(scrapy.Item): - forge_latest = scrapy.Field() - forge_recommended = scrapy.Field() - -class ForgeSpider(CrawlSpider): - name = "ForgeSpider" - allowed_domains = ["minecraftforge.net"] - start_urls = ['http://files.minecraftforge.net'] - - def parse(self, response): - forge = Forge() - forge['versions'] = [] - forge['latest'] = ForgeLatest() - - selector = Selector(response) - rows = selector.xpath('//table[@id="promotions_table"]//tr') - header = rows.pop(0) - for row in rows: - cells = row.xpath('td') - - id = cells[1].xpath('text()').extract() - minecraft = cells[2].xpath('text()').extract() - type = cells[0].xpath('text()') - time = cells[3].xpath('text()') - url = cells[4].xpath('a[text()="Installer"]/@href') - - #if has version - has_version = re.match('(.+)\-.+', ''.join(type.extract())) - if has_version: - download = ForgeVersions() - download['id'] = id - download['minecraft'] = minecraft - download['type'] = 'forge_' + ''.join(type.re('([a-zA-Z]+)')).lower() - download['time'] = time.extract() - download['url'] = url.re('http://adf.ly/\d+/(.+)') - - forge['versions'].append(download) - else: - is_recommended = re.match('Recommended', ''.join(type.extract())) - if is_recommended: - download = ForgeLatest() - forge['latest']['forge_recommended'] = id - else: - download = ForgeLatest() - forge['latest']['forge_latest'] = id - - - return forge - diff --git a/minecraft-server/start-minecraft.sh b/minecraft-server/start-minecraft.sh index 568a7d5b..ba469853 100755 --- a/minecraft-server/start-minecraft.sh +++ b/minecraft-server/start-minecraft.sh @@ -15,22 +15,49 @@ if [ ! -e /data/eula.txt ]; then fi fi -case $VERSION in - LATEST) - export VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'` - ;; - - SNAPSHOT) - export VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.snapshot)'` - ;; -esac - cd /data -if [ ! -e minecraft_server.$VERSION.jar ]; then - echo "Downloading minecraft_server.$VERSION.jar ..." - wget -q https://s3.amazonaws.com/Minecraft.Download/versions/$VERSION/minecraft_server.$VERSION.jar -fi +case $TYPE in + VANILLA) + case $VERSION in + LATEST) + export VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'` + ;; + SNAPSHOT) + export VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.snapshot)'` + ;; + esac + export SERVER="minecraft_server.$VANILLA_VERSION" + + if [ ! -e $SERVER ]; then + echo "Downloading $SERVER ..." + wget -q https://s3.amazonaws.com/Minecraft.Download/versions/$VANILLA_VERSION/$SERVER + fi + ;; + + FORGE) + case $VERSION in + LATEST) + export FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ + | jsawk -n 'out(this.promos.recommended)'` + ;; + SNAPSHOT) + export FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ + | jsawk -n 'out(this.promos.latest)'` + ;; + esac + export FORGE_INSTALLER="forge-$VERSION-$FORGE_VERSION-installer.jar" + export SERVER="forge-$VERSION-$FORGE_VERSION-universal.jar" + + if [ ! -e $SERVER ]; then + echo "Downloading $SERVER ..." + wget -q http://files.minecraftforge.net/maven/net/minecraftforge/forge/$VERSION-$FORGE_VERSION/$FORGE_INSTALLER + fi + + echo "Installing $SERVER" + exec java -jar $FORGE_INSTALLER --installServer + ;; +esac if [ ! -e server.properties ]; then cp /tmp/server.properties . @@ -62,7 +89,7 @@ if [ ! -e server.properties ]; then exit 1 ;; esac - + sed -i "/gamemode\s*=/ c gamemode=$MODE" /data/server.properties fi fi @@ -85,5 +112,5 @@ if [ -n "$ICON" -a ! -e server-icon.png ]; then fi fi -exec java $JVM_OPTS -jar minecraft_server.$VERSION.jar +exec java $JVM_OPTS -jar $SERVER From ddbc7e96186a04c6198ffc0781bc6b5a8e0bcec3 Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Wed, 4 Mar 2015 17:06:51 -0700 Subject: [PATCH 3/7] Add apt-get update to beginning of Dockerfile --- minecraft-server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/minecraft-server/Dockerfile b/minecraft-server/Dockerfile index 84b9accb..6e5cd75d 100644 --- a/minecraft-server/Dockerfile +++ b/minecraft-server/Dockerfile @@ -2,6 +2,7 @@ FROM itzg/ubuntu-openjdk-7 MAINTAINER itzg +RUN apt-get update RUN apt-get install -y wget libmozjs-24-bin imagemagick && apt-get clean RUN update-alternatives --install /usr/bin/js js /usr/bin/js24 100 From 79438e14decd5b2595b9270a411de003406cfc10 Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Wed, 4 Mar 2015 17:47:54 -0700 Subject: [PATCH 4/7] Simplify start-minecraft.sh --- minecraft-server/start-minecraft.sh | 38 ++++++++++++----------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/minecraft-server/start-minecraft.sh b/minecraft-server/start-minecraft.sh index ba469853..8f332b4d 100755 --- a/minecraft-server/start-minecraft.sh +++ b/minecraft-server/start-minecraft.sh @@ -15,19 +15,24 @@ if [ ! -e /data/eula.txt ]; then fi fi +case $VERSION in + LATEST) + VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'` + FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ + | jsawk -n 'out(this.promos.recommended)'` + ;; + SNAPSHOT) + VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.snapshot)'` + FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ + | jsawk -n 'out(this.promos.latest)'` + ;; +esac + cd /data case $TYPE in VANILLA) - case $VERSION in - LATEST) - export VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'` - ;; - SNAPSHOT) - export VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.snapshot)'` - ;; - esac - export SERVER="minecraft_server.$VANILLA_VERSION" + SERVER="minecraft_server.$VANILLA_VERSION.jar" if [ ! -e $SERVER ]; then echo "Downloading $SERVER ..." @@ -36,18 +41,8 @@ case $TYPE in ;; FORGE) - case $VERSION in - LATEST) - export FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ - | jsawk -n 'out(this.promos.recommended)'` - ;; - SNAPSHOT) - export FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ - | jsawk -n 'out(this.promos.latest)'` - ;; - esac - export FORGE_INSTALLER="forge-$VERSION-$FORGE_VERSION-installer.jar" - export SERVER="forge-$VERSION-$FORGE_VERSION-universal.jar" + FORGE_INSTALLER="forge-$VERSION-$FORGE_VERSION-installer.jar" + SERVER="forge-$VERSION-$FORGE_VERSION-universal.jar" if [ ! -e $SERVER ]; then echo "Downloading $SERVER ..." @@ -113,4 +108,3 @@ if [ -n "$ICON" -a ! -e server-icon.png ]; then fi exec java $JVM_OPTS -jar $SERVER - From a252d794f96f34e749aaf2791a4a91d9a1d33110 Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Wed, 4 Mar 2015 18:15:36 -0700 Subject: [PATCH 5/7] Fix bad variable names --- minecraft-server/start-minecraft.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/minecraft-server/start-minecraft.sh b/minecraft-server/start-minecraft.sh index 8f332b4d..97459a98 100755 --- a/minecraft-server/start-minecraft.sh +++ b/minecraft-server/start-minecraft.sh @@ -15,21 +15,21 @@ if [ ! -e /data/eula.txt ]; then fi fi +echo "Checking version information." case $VERSION in LATEST) VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'` - FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ - | jsawk -n 'out(this.promos.recommended)'` + FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json | jsawk -n 'out(this.promos.recommended)'` ;; SNAPSHOT) VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.snapshot)'` - FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json \ - | jsawk -n 'out(this.promos.latest)'` + FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json | jsawk -n 'out(this.promos.latest)'` ;; esac -cd /data +#cd /data +echo "Checking minecraft / forge type information." case $TYPE in VANILLA) SERVER="minecraft_server.$VANILLA_VERSION.jar" @@ -41,16 +41,16 @@ case $TYPE in ;; FORGE) - FORGE_INSTALLER="forge-$VERSION-$FORGE_VERSION-installer.jar" - SERVER="forge-$VERSION-$FORGE_VERSION-universal.jar" + FORGE_INSTALLER="forge-$VANILLA_VERSION-$FORGE_VERSION-installer.jar" + SERVER="forge-$VANILLA_VERSION-$FORGE_VERSION-universal.jar" if [ ! -e $SERVER ]; then - echo "Downloading $SERVER ..." - wget -q http://files.minecraftforge.net/maven/net/minecraftforge/forge/$VERSION-$FORGE_VERSION/$FORGE_INSTALLER + echo "Downloading $FORGE_INSTALLER ..." + wget -q http://files.minecraftforge.net/maven/net/minecraftforge/forge/$VANILLA_VERSION-$FORGE_VERSION/$FORGE_INSTALLER + echo "Installing $SERVER" + exec java -jar $FORGE_INSTALLER --installServer fi - echo "Installing $SERVER" - exec java -jar $FORGE_INSTALLER --installServer ;; esac From 2f4b73d862fca4904668e04878f067329dc30100 Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Wed, 4 Mar 2015 18:17:20 -0700 Subject: [PATCH 6/7] Remove commented out cd /data from testing --- minecraft-server/start-minecraft.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minecraft-server/start-minecraft.sh b/minecraft-server/start-minecraft.sh index 97459a98..48db73c0 100755 --- a/minecraft-server/start-minecraft.sh +++ b/minecraft-server/start-minecraft.sh @@ -27,7 +27,7 @@ case $VERSION in ;; esac -#cd /data +cd /data echo "Checking minecraft / forge type information." case $TYPE in From df09adbfd7cf0c63e807dbebd57662ad35da9e3d Mon Sep 17 00:00:00 2001 From: "Daniel L. Polanco" Date: Wed, 4 Mar 2015 20:07:37 -0700 Subject: [PATCH 7/7] Add non-working code to work with Goeff --- minecraft-server/start-minecraft.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/minecraft-server/start-minecraft.sh b/minecraft-server/start-minecraft.sh index 48db73c0..d3cc62f9 100755 --- a/minecraft-server/start-minecraft.sh +++ b/minecraft-server/start-minecraft.sh @@ -50,7 +50,6 @@ case $TYPE in echo "Installing $SERVER" exec java -jar $FORGE_INSTALLER --installServer fi - ;; esac