[es] Add note about increasing mmap count

Fixes #119
* also switch/upgrade base to openjdk 8u111

cassandra/Dockerfile

@@ -1,11 +1,11 @@
-FROM itzg/ubuntu-openjdk-7
+FROM java:openjdk-8u72-jdk
 
 MAINTAINER itzg
 
-ENV CASSANDRA_VERSION 2.2.5
+ENV CASSANDRA_VERSION 2.2.8
 
-RUN wget -qO /tmp/apache-cassandra.tgz http://mirrors.ibiblio.org/apache/cassandra/$CASSANDRA_VERSION/apache-cassandra-$CASSANDRA_VERSION-bin.tar.gz
-RUN tar -C /opt -zxvf /tmp/apache-cassandra.tgz && \
+ADD http://apache.mirrors.pair.com/cassandra/$CASSANDRA_VERSION/apache-cassandra-$CASSANDRA_VERSION-bin.tar.gz /tmp/apache-cassandra.tgz
+RUN tar -C /opt -zxf /tmp/apache-cassandra.tgz && \
   rm /tmp/apache-cassandra.tgz
 
 

cassandra/cassandra.in.sh

@@ -1,19 +1,17 @@
 #!/bin/bash
 
-sed -i -e 's/log4j.rootLogger=.*/log4j.rootLogger=INFO,stdout/' $CASSANDRA_HOME/conf/log4j-server.properties
-
 cassYml=$CASSANDRA_HOME/conf/cassandra.yaml
 
 privateAddr=$(hostname -i)
 
 seeds=${SEEDS:-${PUBLISH_AS:-$privateAddr}}
 
-sed -i -e "s/- seeds:.*/- seeds: \"$seeds\"/" $cassYml
-
-sed -i -e "s/listen_address:.*/listen_address: $privateAddr/" $cassYml
-sed -i -e "s/rpc_address:.*/rpc_address: $privateAddr/" $cassYml
-
-sed -i -e "s#- /var/lib/cassandra/data#- $CASSANDRA_DATA#" $cassYml
+sed -i -e "s/- seeds:.*/- seeds: \"$seeds\"/" \
+  -e "s/listen_address:.*/listen_address: $privateAddr/" \
+  -e "s/rpc_address:.*/rpc_address: $privateAddr/" \
+  -e "s/start_rpc:.*/start_rpc: true/" \
+  -e "s#- /var/lib/cassandra/data#- $CASSANDRA_DATA#" \
+  $cassYml
 
 if [ -n "$PUBLISH_AS" ]; then
   sed -i -e "s/\(\s*#\)\?\s*broadcast_address:.*/broadcast_address: $PUBLISH_AS/" $cassYml
@@ -24,4 +22,3 @@ cp -rn $CASSANDRA_HOME/conf/* $CASSANDRA_CONF
 
 # source the original
 . $CASSANDRA_HOME/bin/orig.cassandra.in.sh
-

elasticsearch/Dockerfile

@@ -1,28 +1,32 @@
-FROM java:openjdk-8u72-jdk
+FROM openjdk:8u111-jre-alpine
 
 MAINTAINER itzg
 
-ENV ES_VERSION 2.3.0
+RUN apk -U add bash
 
-RUN wget -qO /tmp/es.tgz https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.tar.gz && \
-  cd /usr/share && \
+ENV ES_VERSION=5.1.1
+
+ADD https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.tar.gz /tmp/es.tgz
+RUN cd /usr/share && \
   tar xf /tmp/es.tgz && \
   rm /tmp/es.tgz
 
-ENV ES_HOME /usr/share/elasticsearch-$ES_VERSION
-RUN useradd -d $ES_HOME -M -r elasticsearch && \
-  chown -R elasticsearch: $ES_HOME
-
-RUN mkdir /data /conf && touch /data/.CREATED /conf/.CREATED && chown -R elasticsearch: /data /conf
-VOLUME ["/data","/conf"]
-
-ADD start /start
-
-WORKDIR $ES_HOME
-USER elasticsearch
-
 EXPOSE 9200 9300
 
-ENV OPTS=-Dnetwork.host=_non_loopback_
+ENV ES_HOME=/usr/share/elasticsearch-$ES_VERSION \
+    DEFAULT_ES_USER=elasticsearch \
+    DISCOVER_TRANSPORT_IP=eth0 \
+    DISCOVER_HTTP_IP=eth0 \
+    ES_JAVA_OPTS="-Xms1g -Xmx1g"
+
+RUN adduser -S -s /bin/sh $DEFAULT_ES_USER
+
+VOLUME ["/data","/conf"]
+
+WORKDIR $ES_HOME
+
+COPY java.policy /usr/lib/jvm/java-1.8-openjdk/jre/lib/security/
+COPY start /start
+COPY log4j2.properties $ES_HOME/config/
 
 CMD ["/start"]

elasticsearch/README.md

@@ -1,17 +1,17 @@
-This Docker image provides an easily configurable Elasticsearch node. Via
-port mappings, it is easy to create an arbitrarily sized cluster of
-nodes. As long as the versions match, you can mix-and-match "real"
-Elasticsearch nodes with container-ized ones.
+This Docker image provides an easily configurable Elasticsearch node. Via port mappings, it is easy to create an arbitrarily sized cluster of nodes. As long as the versions match, you can mix-and-match "real" Elasticsearch nodes with container-ized ones.
+
+# NOTE for use on Linux hosts
+
+Elasticsearch 5.x requires that the virtual memory mmap count is set sufficiently for stable,
+production use. [Refer to this guide for more information](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html).
 
 # Basic Usage
 
-To start an Elasticsearch data node that listens on the standard ports on
-your host's network interface:
+To start an Elasticsearch data node that listens on the standard ports on your host's network interface:
 
     docker run -d -p 9200:9200 -p 9300:9300 itzg/elasticsearch
 
-You'll then be able to connect to the Elasticsearch HTTP interface to confirm
-it's alive:
+You'll then be able to connect to the Elasticsearch HTTP interface to confirm it's alive:
 
 http://DOCKERHOST:9200/
 
@@ -28,16 +28,15 @@ http://DOCKERHOST:9200/
       "tagline" : "You Know, for Search"
     }
 
-Where `DOCKERHOST` would be the actual hostname of your host running
-Docker.
+Where `DOCKERHOST` would be the actual hostname of your host running Docker.
 
 # Simple, multi-node cluster
 
 To run a multi-node cluster (3-node in this example) on a single Docker machine use:
 
-    docker run -d --name es0 -p 9200:9200                    es
-    docker run -d --name es1 --link es0 -e UNICAST_HOSTS=es0 es
-    docker run -d --name es2 --link es0 -e UNICAST_HOSTS=es0 es
+    docker run -d --name es0 -p 9200:9200                    itzg/elasticsearch
+    docker run -d --name es1 --link es0 -e UNICAST_HOSTS=es0 itzg/elasticsearch
+    docker run -d --name es2 --link es0 -e UNICAST_HOSTS=es0 itzg/elasticsearch
 
 
 and then check the cluster health, such as http://192.168.99.100:9200/_cluster/health?pretty
@@ -55,17 +54,25 @@ and then check the cluster health, such as http://192.168.99.100:9200/_cluster/h
       "unassigned_shards" : 0
     }
 
-# Configuration
+# Configuration Summary
 
-The following configuration options are specified using `docker run`
-environment variables (`-e`) like
+## Ports
+
+* `9200` - HTTP REST
+* `9300` - Native transport
+
+## Volumes
+
+* `/data` - location of `path.data`
+* `/conf` - location of `path.conf`
+
+# Configuration Details
+
+The following configuration options are specified using `docker run` environment variables (`-e`) like
 
     docker run ... -e NAME=VALUE ... itzg/elasticsearch
 
-Since Docker's `-e` settings are baked into the container definition, this image provides an
-extra feature to change any of the settings below for an existing container. Either
-create/edit the file `env` in the `/conf` volume mapping or edit within the running container's
-context using:
+Since Docker's `-e` settings are baked into the container definition, this image provides an extra feature to change any of the settings below for an existing container. Either create/edit the file `env` in the `/conf` volume mapping or edit within the running container's context using:
 
     docker exec -it CONTAINER_ID vi /conf/env
 
@@ -77,8 +84,7 @@ The contents of the `/conf/env` file are standard shell
 
 entries where `NAME` is one of the variables described below.
 
-Configuration options not explicitly supported below can be specified via the `OPTS` environment variable. For
-example, by default `OPTS` is set with
+Configuration options not explicitly supported below can be specified via the `OPTS` environment variable. For example, by default `OPTS` is set with
 
     OPTS=-Dnetwork.bind_host=_non_loopback_
 
@@ -87,17 +93,13 @@ port mapping out from the container_.
 
 ## Cluster Name
 
-If joining a pre-existing cluster, then you may need to specify a cluster name
-different than the default "elasticsearch":
+If joining a pre-existing cluster, then you may need to specify a cluster name different than the default "elasticsearch":
 
     -e CLUSTER=dockers
 
 ## Zen Unicast Hosts
 
-When joining a multi-physical-host cluster, multicast may not be supported
-on the physical network. In that case, your node can reference specific one or more hosts in
-the cluster via the
-[Zen Unicast Hosts](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#unicast) capability as a comma-separated list of `HOST:PORT` pairs:
+When joining a multi-physical-host cluster, multicast may not be supported on the physical network. In that case, your node can reference specific one or more hosts in the cluster via the [Zen Unicast Hosts](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#unicast) capability as a comma-separated list of `HOST:PORT` pairs:
 
     -e UNICAST_HOSTS=HOST:PORT[,HOST:PORT]
 
@@ -107,8 +109,7 @@ such as
 
 ## Plugins
 
-You can install one or more plugins before startup by passing a comma-separated
-list of plugins.
+You can install one or more plugins before startup by passing a comma-separated list of plugins.
 
     -e PLUGINS=ID[,ID]
 
@@ -120,10 +121,7 @@ Many more plugins [are available here](http://www.elasticsearch.org/guide/en/ela
 
 ## Publish As
 
-Since the container gives the Elasticsearch software an isolated perspective
-of its networking, it will most likely advertise its published address with
-a container-internal IP address. This can be overridden with a physical networking
-name and port using:
+Since the container gives the Elasticsearch software an isolated perspective of its networking, it will most likely advertise its published address with a container-internal IP address. This can be overridden with a physical networking name and port using:
 
     -e PUBLISH_AS=DOCKERHOST:9301
 
@@ -132,7 +130,89 @@ than the cosmetic weirdness in the logs, Elasticsearch seems to be quite toleran
 
 ## Node Name
 
-Rather than use the randomly assigned node name, you can indicate a specific
-one using:
+Rather than use the randomly assigned node name, you can indicate a specific one using:
 
     -e NODE_NAME=Docker
+
+## Node Type
+
+If you refer to [the Node section](https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-node.html)
+of the Elasticsearch reference guide, you'll find that there's three main types of nodes: master-eligible, data, and client.
+
+In larger clusters it is important to dedicate a small number (>= 3) of master nodes. There are also cases where a large cluster may need dedicated gateway nodes that are neither master nor data nodes and purely operate as "smart routers" and have large amounts of CPU and memory to handle client requests and search-reduce.
+
+To simplify all that, this image provides a `TYPE` variable to let you amongst these combinations. The choices are:
+
+* (not set, the default) : the default node type which is both master-eligible and a data node
+* `MASTER` : master-eligible, but holds no data. It is good to have three or more of these in a
+large cluster
+* `DATA` (or `NON_MASTER`) : holds data and serves search/index requests. Scale these out for elastic-y goodness.
+* `GATEWAY` : only operates as a client node or a "smart router". These are the ones whose HTTP port 9200 will need to be exposed
+
+A [Docker Compose](https://docs.docker.com/compose/overview/) file will serve as a good example of these three node types:
+
+```
+version: '2'
+
+services:
+  gateway:
+    image: itzg/elasticsearch
+    environment:
+      UNICAST_HOSTS: master
+      TYPE: GATEWAY
+    ports:
+      - "9200:9200"
+
+  master:
+    image: itzg/elasticsearch
+    environment:
+      UNICAST_HOSTS: gateway
+      TYPE: MASTER
+      MIN_MASTERS: 2
+
+  data:
+    image: itzg/elasticsearch
+    environment:
+      UNICAST_HOSTS: master,gateway
+      TYPE: DATA
+```
+
+## Minimum Master Nodes
+
+In combination with the `TYPE` variable above, you will also want to configure the minimum master nodes to [avoid split-brain](https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-node.html#split-brain) during network outages.
+
+The minimum, which can be calculated as `(master_eligible_nodes / 2) + 1`, can be set with the `MIN_MASTERS` variable.
+
+Using the Docker Compose file above, a value of `2` is appropriate when scaling the cluster to 3 master nodes:
+
+    docker-compose scale master=3
+
+## Auto transport/http discovery with Swarm Mode
+
+When using Docker Swarm mode (starting with 1.12), the overlay and ingress network interfaces are assigned
+multiple IP addresses. As a result, it creates confusion for the transport publish logic even when using
+the special value `_eth0_`.
+
+To resolve this, add
+
+    -e DISCOVER_TRANSPORT_IP=eth0
+
+replacing `eth0` with another interface within the container, if needed.
+
+The same can be done for publish/binding of the http module by adding:
+
+    -e DISCOVER_HTTP_IP=eth2
+
+## Heap size and other JVM options
+
+By default this image will run Elasticsearch with a Java heap size of 1 GB. If that value
+or any other JVM options need to be adjusted, then replace the `ES_JAVA_OPTS`
+environment variable.
+
+For example, this would allow for the use of 16 GB of heap:
+
+    -e ES_JAVA_OPTS="-Xms16g -Xmx16g"
+
+Refer to [this page](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html)
+for more information about why both the minimum and maximum sizes were set to
+the same value.

elasticsearch/java.policy

@@ -0,0 +1,6 @@
+grant {
+  // JMX Java Management eXtensions
+  permission javax.management.MBeanTrustPermission "register";
+  permission javax.management.MBeanServerPermission "createMBeanServer";
+  permission javax.management.MBeanPermission "-#-[-]", "queryNames";
+};

elasticsearch/log4j2.properties

@@ -0,0 +1,74 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs}.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n
+appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+#rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 1GB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 4
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+#logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = RollingFile
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.layout.type = PatternLayout
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.policies.type = Policies
+appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_search_slowlog_rolling.policies.time.interval = 1
+appender.index_search_slowlog_rolling.policies.time.modulate = true
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = console
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = console
+logger.index_indexing_slowlog.additivity = false

elasticsearch/start

@@ -1,5 +1,98 @@
 #!/bin/sh
 
+pre_checks() {
+  mmc=$(sysctl vm.max_map_count|sed 's/.*= //')
+  if [[ $mmc -lt 262144 ]]; then
+    echo "
+ERROR: As of 5.0.0 Elasticsearch requires increasing mmap counts.
+Refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
+"
+    exit 1
+  fi
+}
+
+discoverIpFromLink() {
+  dev=$1
+  mode=$2
+  ip=`ipaddr show dev $dev scope global|awk '$1 == "inet" { if (!match($2,"/32")) { gsub("/.*","",$2) ; print $2 } }'`
+  echo "Discovered $mode address $ip for $dev"
+  OPTS="$OPTS -E $mode.host=$ip"
+}
+
+setup_clustering() {
+
+  if [ -n "$CLUSTER" ]; then
+    OPTS="$OPTS -E cluster.name=$CLUSTER"
+    if [ -n "$CLUSTER_FROM" ]; then
+      if [ -d /data/$CLUSTER_FROM -a ! -d /data/$CLUSTER ]; then
+        echo "Performing cluster data migration from $CLUSTER_FROM to $CLUSTER"
+        mv /data/$CLUSTER_FROM /data/$CLUSTER
+      fi
+    fi
+  fi
+
+  if [ -n "$NODE_NAME" ]; then
+    OPTS="$OPTS -E node.name=$NODE_NAME"
+  fi
+
+  if [ -n "$MULTICAST" ]; then
+    OPTS="$OPTS -E discovery.zen.ping.multicast.enabled=$MULTICAST"
+  fi
+
+  if [ -n "$UNICAST_HOSTS" ]; then
+    OPTS="$OPTS -E discovery.zen.ping.unicast.hosts=$UNICAST_HOSTS"
+  fi
+
+  if [ -n "$PUBLISH_AS" ]; then
+    OPTS="$OPTS -E transport.publish_host=$(echo $PUBLISH_AS | awk -F: '{print $1}')"
+    OPTS="$OPTS -E transport.publish_port=$(echo $PUBLISH_AS | awk -F: '{if ($2) print $2; else print 9300}')"
+  fi
+
+  if [ -n "$MIN_MASTERS" ]; then
+    OPTS="$OPTS -E discovery.zen.minimum_master_nodes=$MIN_MASTERS"
+  fi
+
+}
+
+install_plugins() {
+
+  if [ -n "$PLUGINS" ]; then
+    for p in $(echo $PLUGINS | awk -v RS=, '{print}')
+    do
+      echo "Installing the plugin $p"
+      $ES_HOME/bin/elasticsearch-plugin install $p
+    done
+  else
+    mkdir -p $ES_HOME/plugins
+  fi
+}
+
+setup_personality() {
+
+  if [ -n "$TYPE" ]; then
+    case $TYPE in
+      MASTER)
+        OPTS="$OPTS -E node.master=true -E node.data=false"
+        ;;
+
+      GATEWAY)
+        OPTS="$OPTS -E node.master=false -E node.data=false"
+        ;;
+
+      DATA|NON_MASTER)
+        OPTS="$OPTS -E node.master=false -E node.data=true"
+        ;;
+
+      *)
+        echo "Unknown node type. Please use MASTER|GATEWAY|DATA|NON_MASTER"
+        exit 1
+    esac
+  fi
+
+}
+
+pre_checks
+
 if [ -f /conf/env ]; then
   . /conf/env
 fi
@@ -8,50 +101,32 @@ if [ ! -e /conf/elasticsearch.* ]; then
   cp $ES_HOME/config/elasticsearch.yml /conf
 fi
 
-if [ ! -e /conf/logging.* ]; then
-  cp $ES_HOME/config/logging.yml /conf
+if [ ! -e /conf/log4j2.properties ]; then
+  cp $ES_HOME/config/log4j2.properties /conf
 fi
 
-OPTS="$OPTS -Des.path.conf=/conf \
-  -Des.path.data=/data \
-  -Des.path.logs=/data \
-  -Des.transport.tcp.port=9300 \
-  -Des.http.port=9200"
+OPTS="$OPTS \
+  -E path.conf=/conf \
+  -E path.data=/data \
+  -E path.logs=/data \
+  -E transport.tcp.port=9300 \
+  -E http.port=9200"
 
-if [ -n "$CLUSTER" ]; then
-  OPTS="$OPTS -Des.cluster.name=$CLUSTER"
-  if [ -n "$CLUSTER_FROM" ]; then
-    if [ -d /data/$CLUSTER_FROM -a ! -d /data/$CLUSTER ]; then
-      echo "Performing cluster data migration from $CLUSTER_FROM to $CLUSTER"
-      mv /data/$CLUSTER_FROM /data/$CLUSTER
-    fi
-  fi
-fi
+discoverIpFromLink $DISCOVER_TRANSPORT_IP transport
+discoverIpFromLink $DISCOVER_HTTP_IP http
 
-if [ -n "$NODE_NAME" ]; then
-  OPTS="$OPTS -Des.node.name=$NODE_NAME"
-fi
+setup_personality
+setup_clustering
+install_plugins
 
-if [ -n "$MULTICAST" ]; then
-  OPTS="$OPTS -Des.discovery.zen.ping.multicast.enabled=$MULTICAST"
-fi
-
-if [ -n "$UNICAST_HOSTS" ]; then
-  OPTS="$OPTS -Des.discovery.zen.ping.unicast.hosts=$UNICAST_HOSTS"
-fi
-
-if [ -n "$PUBLISH_AS" ]; then
-  OPTS="$OPTS -Des.transport.publish_host=$(echo $PUBLISH_AS | awk -F: '{print $1}')"
-  OPTS="$OPTS -Des.transport.publish_port=$(echo $PUBLISH_AS | awk -F: '{if ($2) print $2; else print 9300}')"
-fi
-
-if [ -n "$PLUGINS" ]; then
-  for p in $(echo $PLUGINS | awk -v RS=, '{print}')
-  do
-    echo "Installing the plugin $p"
-    $ES_HOME/bin/plugin install $p
-  done
-fi
+mkdir -p /conf/scripts
 
 echo "Starting Elasticsearch with the options $OPTS"
-$ES_HOME/bin/elasticsearch $OPTS
+CMD="$ES_HOME/bin/elasticsearch $OPTS"
+if [ `id -u` = 0 ]; then
+  echo "Running as non-root..."
+  chown -R $DEFAULT_ES_USER /data /conf
+  su -c "$CMD" $DEFAULT_ES_USER
+else
+  $CMD
+fi

gitblit/Dockerfile

@@ -1,30 +1,26 @@
-FROM itzg/ubuntu-openjdk-7
+FROM java:8
  
 MAINTAINER itzg
 
-ENV APT_GET_UPDATE 2014-07-19
+ENV GITBLIT_VERSION 1.7.1
  
-RUN apt-get update
-RUN apt-get -y install wget tar
+RUN wget -qO /tmp/gitblit.tgz http://dl.bintray.com/gitblit/releases/gitblit-$GITBLIT_VERSION.tar.gz
  
-ENV GITBLIT_VERSION 1.6.0
- 
-RUN wget -O /tmp/gitblit.tgz http://dl.bintray.com/gitblit/releases/gitblit-$GITBLIT_VERSION.tar.gz
- 
-RUN mkdir /opt/gitblit
-RUN tar -C /opt/gitblit -xvf /tmp/gitblit.tgz
-RUN rm /tmp/gitblit.tgz
+RUN tar -C /opt -xvf /tmp/gitblit.tgz && \
+    rm /tmp/gitblit.tgz
  
 VOLUME ["/data"]
  
 ADD start.sh /start
  
-WORKDIR /opt/gitblit
-ENV GITBLIT_PATH /opt/gitblit
-ENV GITBLIT_HTTPS_PORT 443
-ENV GITBLIT_HTTP_PORT 80
-ENV GITBLIT_BASE_FOLDER /data
+ENV GITBLIT_PATH=/opt/gitblit-${GITBLIT_VERSION} \
+    GITBLIT_HTTPS_PORT=443 \
+    GITBLIT_HTTP_PORT=80 \
+    GITBLIT_BASE_FOLDER=/data \
+    GITBLIT_ADMIN_USER=admin \
+    GITBLIT_INITIAL_REPO=
+WORKDIR $GITBLIT_PATH
  
 EXPOSE 80 443
  
-CMD ["/start"]
+ENTRYPOINT ["/start"]

gitblit/README.md

@@ -1,5 +1,46 @@
-Provides a ready-to-use instance of [GitBlit](http://gitblit.com/). In order to allow for 
-future upgrades run the container with a volume mount of `/data`, such as:
+Provides a ready-to-use instance of [GitBlit](http://gitblit.com/).
 
-    docker run -d -p 80:80 p 443:443 -v /tmp/gitblit-data:/data --name gitblit itzg/gitblit
+## Basic usage
 
+Start the GitBlit container using
+
+    docker run -d -p 80:80 -p 443:443 --name gitblit itzg/gitblit
+
+Access its web interface at the mapped HTTP (80) or HTTPS (443) port of the
+Docker host. Login with the default credentials __admin__ / __admin__ .
+
+
+## Data volume
+
+In order to allow for future upgrades, run the container with a volume mount of `/data`, such as:
+
+    -v /tmp/gitblit-data:/data
+
+## Initial repository creation
+
+As a convenience for cluster configuration management with git
+(such as with [Spring Cloud Config](https://cloud.spring.io/spring-cloud-config/)),
+you may specify the name of an initial repository to be owned by the 'admin' user.
+This can be enabled by passing the name of that repository via the environment
+variable `GITBLIT_INITIAL_REPO`, such as
+
+    -e GITBLIT_INITIAL_REPO=default
+
+## Create repositories with content
+
+In addition to the approach above, you can push repostories with existing
+content by attaching them to sub-directories of `/repos`, such as
+
+    docker run -v $HOME/git/example:/repos/example ...
+
+## Custom configuration
+
+You can add or override any of the `*.properties` files for configuring GitBlit,
+typically `gitblit.properties`, by placing those files in a volume attached at
+`/config`, such as
+
+    -v $(pwd)/extra-config:/config
+
+The property files in that configuration directory will be renamed with the
+suffix `.applied` to avoid overwriting manually modified configuration on
+the next container startup.

gitblit/start.sh

@@ -1,10 +1,103 @@
-#!/bin/sh
+#!/bin/bash
 
-contents=`ls $GITBLIT_BASE_FOLDER|wc -l`
+apply_base_data() {
+  contents=`ls $GITBLIT_BASE_FOLDER|wc -l`
 
-if [ $contents = "0" ]; then
-  cp -r $GITBLIT_PATH/data/* $GITBLIT_BASE_FOLDER
+  if [ $contents = "0" ]; then
+    cp -r $GITBLIT_PATH/data/* $GITBLIT_BASE_FOLDER
+  fi
+}
+
+apply_config() {
+    cp -rf /config/* $GITBLIT_BASE_FOLDER
+}
+
+create_repo() {
+  local repo_dir=$GITBLIT_BASE_FOLDER/git/$1.git
+  mkdir -p $repo_dir
+  cd $repo_dir
+
+  git init --bare
+
+  echo "
+[gitblit]
+        description =
+        originRepository =
+        owner = $GITBLIT_ADMIN_USER
+        acceptNewPatchsets = true
+        acceptNewTickets = true
+        mergeTo = master
+        useIncrementalPushTags = false
+        allowForks = true
+        accessRestriction = PUSH
+        authorizationControl = AUTHENTICATED
+        verifyCommitter = false
+        showRemoteBranches = false
+        isFrozen = false
+        skipSizeCalculation = false
+        skipSummaryMetrics = false
+        federationStrategy = FEDERATE_THIS
+        isFederated = false
+        gcThreshold =
+        gcPeriod = 0
+" >> config
+
+  git config --replace-all core.logallrefupdates false
+
+  echo "
+CREATING repository '$1' with:
+* read/clone access for all
+* push access for authenticated users"
+
+  RET="file://$repo_dir"
+}
+
+apply_repos() {
+  for rdir in /repos/*; do
+    if [ -e $rdir/.git ]; then
+      r=$(basename $rdir)
+      create_repo $r
+      local url=$RET
+      cd $rdir
+      echo "* pushed existing content"
+      git push --all $url
+    fi
+
+  done
+}
+
+create_initial_repo() {
+  if [ -d $GITBLIT_INITIAL_REPO ]; then
+    return
+  fi
+
+  create_repo $GITBLIT_INITIAL_REPO
+}
+
+shopt -s nullglob
+if [ ! -f /var/local/gitblit_firststart ]; then
+    FIRSTSTART=1
+else
+    FIRSTSTART=0
 fi
 
-$JAVA_HOME/bin/java -jar $GITBLIT_PATH/gitblit.jar --httpsPort $GITBLIT_HTTPS_PORT --httpPort $GITBLIT_HTTP_PORT --baseFolder $GITBLIT_BASE_FOLDER
+if [ $FIRSTSTART = 1 ]; then
+  apply_base_data
 
+  echo "
+Applying configuration from /config
+"
+  apply_config
+  touch /var/local/gitblit_firststart
+fi
+
+
+if [[ -n $GITBLIT_INITIAL_REPO ]]; then
+  create_initial_repo
+fi
+apply_repos
+
+cd $GITBLIT_PATH
+$JAVA_HOME/bin/java -jar $GITBLIT_PATH/gitblit.jar \
+  --httpsPort $GITBLIT_HTTPS_PORT --httpPort $GITBLIT_HTTP_PORT \
+  --baseFolder $GITBLIT_BASE_FOLDER

jenkins/Dockerfile

@@ -1,11 +1,17 @@
-FROM java:openjdk-8u72-jdk
+FROM java:openjdk-8u102-jdk
 
 MAINTAINER itzg
 
-ADD download-and-start.sh /download-and-start
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y \
+  graphviz \
+  && apt-get clean
 
-ENV JENKINS_HOME /data
-VOLUME ["/data"]
+ENV JENKINS_HOME=/data
+
+VOLUME ["/data", "/root", "/opt/jenkins"]
 EXPOSE 8080 38252
 
-CMD ["/download-and-start"]
+COPY download-and-start.sh /opt/download-and-start
+
+CMD ["/opt/download-and-start"]

jenkins/README.md

@@ -18,21 +18,6 @@ would be:
 
     ID=$(docker run -d -p 8080:8080 -v /SOME_HOST_DIR:/data itzg/jenkins
 
-# Entering the container to perform manual config
-
-As with any Docker container, you can run a shell within the running container:
-
-    docker exec -it $ID bash
-
-A more realistic example is installing git, openjdk-7-jdk, etc:
-
-    docker exec $ID apt-get update
-    docker exec $ID apt-get install -y git openjdk-7-jdk
-
-and then Configure the JDK in Jenkins:
-
-![](http://i.imgur.com/HVetwKc.png)
-
 # Enabling Jenkins slave agents
 
 By default, Jenkins will pick a random port to allow slave nodes launched
@@ -47,3 +32,20 @@ Launch your Jenkins container using
 and configure the port in the Global Security settings:
 
 ![](http://i.imgur.com/PhQiEHy.png)
+
+# Image Parameters
+
+## Volumes
+
+* `/data` - a majority of the Jenkins content is maintained here, such as workspaces
+* `/root` - some tools, such as Maven, utilize the home directory for default repository storage
+* `/opt/jenkins` - the installed distribution is expanded here
+
+## Ports
+
+* `8080` - for the web UI
+* `38252` - for slave incoming JMX access
+
+## Environment Variables
+
+* `JENKINS_OPTS` - passed to the initial Java invocation of Jenkins

jenkins/download-and-start.sh

@@ -26,4 +26,4 @@ fi
 
 
 cd /opt/jenkins
-exec java -jar jenkins.war
+exec java $JENKINS_OPTS -jar jenkins.war

kibana/Dockerfile

@@ -1,13 +1,14 @@
-FROM itzg/ubuntu-openjdk-7
+FROM openjdk:8u111-jre
 
 MAINTAINER itzg
 
-ENV KIBANA_VERSION 4.1.1
+ENV KIBANA_VERSION 5.1.1
 
-RUN wget -q -O /tmp/kibana.tgz https://download.elasticsearch.org/kibana/kibana/kibana-${KIBANA_VERSION}-linux-x64.tar.gz
+ADD https://artifacts.elastic.co/downloads/kibana/kibana-${KIBANA_VERSION}-linux-x86_64.tar.gz /tmp/kibana.tgz
 
 RUN tar -C /opt -xzf /tmp/kibana.tgz && rm /tmp/kibana.tgz
-ENV KIBANA_HOME /opt/kibana-$KIBANA_VERSION-linux-x64
+
+ENV KIBANA_HOME /opt/kibana-$KIBANA_VERSION-linux-x86_64
 
 # Simplify for cross-container
 ENV ES_URL http://es:9200

kibana/start.sh

@@ -1,6 +1,5 @@
-#!/bin/bash
+#!/bin/sh
 
-OPTS="-e $ES_URL"
+OPTS="-e $ES_URL -H $HOSTNAME"
 
 exec bin/kibana $OPTS
-

minecraft-server/Dockerfile

@@ -2,47 +2,42 @@ FROM java:8
 
 MAINTAINER itzg
 
-ENV APT_GET_UPDATE 2015-10-03
+ENV APT_GET_UPDATE 2016-04-23
 RUN apt-get update
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \
-  libmozjs-24-bin \
   imagemagick \
   lsof \
   nano \
   sudo \
   vim \
+  jq \
   && apt-get clean
-RUN update-alternatives --install /usr/bin/js js /usr/bin/js24 100
 
-RUN wget -O /usr/bin/jsawk https://github.com/micha/jsawk/raw/master/jsawk
-RUN chmod +x /usr/bin/jsawk
-RUN useradd -M -s /bin/false --uid 1000 minecraft \
+RUN useradd -s /bin/false --uid 1000 minecraft \
   && mkdir /data \
   && mkdir /config \
   && mkdir /mods \
   && mkdir /plugins \
-  && chown minecraft:minecraft /data /config /mods /plugins
+  && mkdir /home/minecraft \
+  && chown minecraft:minecraft /data /config /mods /plugins /home/minecraft
 
-EXPOSE 25565
+EXPOSE 25565 25575
 
+ADD https://github.com/itzg/restify/releases/download/1.0.3/restify_linux_amd64 /usr/local/bin/restify
 COPY start.sh /start
 COPY start-minecraft.sh /start-minecraft
+COPY mcadmin.jq /usr/share
+RUN chmod +x /usr/local/bin/*
 
-VOLUME ["/data"]
-VOLUME ["/mods"]
-VOLUME ["/config"]
-VOLUME ["/plugins"]
+VOLUME ["/data","/mods","/config","/plugins","/home/minecraft"]
 COPY server.properties /tmp/server.properties
 WORKDIR /data
 
-CMD [ "/start" ]
+ENTRYPOINT [ "/start" ]
 
-# Special marker ENV used by MCCY management tool
-ENV MC_IMAGE=YES
-
-ENV UID=1000 GID=1000
-ENV MOTD A Minecraft Server Powered by Docker
-ENV JVM_OPTS -Xmx1024M -Xms1024M
-ENV TYPE=VANILLA VERSION=LATEST FORGEVERSION=RECOMMENDED LEVEL=world PVP=true DIFFICULTY=easy \
-  LEVEL_TYPE=DEFAULT GENERATOR_SETTINGS= WORLD= MODPACK=
+ENV UID=1000 GID=1000 \
+    MOTD="A Minecraft Server Powered by Docker" \
+    JVM_OPTS="-Xmx1024M -Xms1024M" \
+    TYPE=VANILLA VERSION=LATEST FORGEVERSION=RECOMMENDED LEVEL=world PVP=true DIFFICULTY=easy \
+    LEVEL_TYPE=DEFAULT GENERATOR_SETTINGS= WORLD= MODPACK= ONLINE_MODE=TRUE CONSOLE=true

minecraft-server/README.md

@@ -1,6 +1,6 @@
 
 [![Docker Pulls](https://img.shields.io/docker/pulls/itzg/minecraft-server.svg)](https://hub.docker.com/r/itzg/minecraft-server/)
-
+[![Docker Stars](https://img.shields.io/docker/stars/itzg/minecraft-server.svg?maxAge=2592000)](https://hub.docker.com/r/itzg/minecraft-server/)
 
 This docker image provides a Minecraft Server that will automatically download the latest stable
 version at startup. You can also run/upgrade to any specific version or the
@@ -147,7 +147,7 @@ There are two additional volumes that can be mounted; `/mods` and `/config`.
 Any files in either of these filesystems will be copied over to the main
 `/data` filesystem before starting Minecraft.
 
-This works well if you want to have a common set of modules in a separate 
+This works well if you want to have a common set of modules in a separate
 location, but still have multiple worlds with different server requirements
 in either persistent volumes or a downloadable archive.
 
@@ -155,14 +155,21 @@ in either persistent volumes or a downloadable archive.
 
 Enable Bukkit/Spigot server mode by adding a `-e TYPE=BUKKIT -e VERSION=1.8` or `-e TYPE=SPIGOT -e VERSION=1.8` to your command-line.
 
-The VERSION option should be set to 1.8, as this is the only version of CraftBukkit and Spigot currently
-available.  The latest build in this branch will be used.
-
-    $ docker run -d -v /path/on/host:/data \
+    docker run -d -v /path/on/host:/data \
         -e TYPE=SPIGOT -e VERSION=1.8 \
         -p 25565:25565 -e EULA=TRUE --name mc itzg/minecraft-server
 
-You can install Bukkit plugins in two ways.
+You can build spigot from source by adding `-e BUILD_FROM_SOURCE=true`
+
+__NOTE: to avoid pegging the CPU when running Spigot,__ you will need to
+pass `--noconsole` at the very end of the command line and not use `-it`. For example,
+
+    docker run -d -v /path/on/host:/data \
+        -e TYPE=SPIGOT -e VERSION=1.8 \
+        -p 25565:25565 -e EULA=TRUE --name mc itzg/minecraft-server --noconsole
+
+
+You can install Bukkit plugins in two ways...
 
 ### Using the /data volume
 
@@ -198,7 +205,62 @@ There is one additional volume that can be mounted; `/plugins`.
 Any files in this filesystem will be copied over to the main
 `/data/plugins` filesystem before starting Minecraft.
 
-This works well if you want to have a common set of plugins in a separate 
+This works well if you want to have a common set of plugins in a separate
+location, but still have multiple worlds with different server requirements
+in either persistent volumes or a downloadable archive.
+
+## Running a PaperSpigot server
+
+Enable PaperSpigot server mode by adding a `-e TYPE=PAPER -e VERSION=1.9.4` to your command-line.
+
+    docker run -d -v /path/on/host:/data \
+        -e TYPE=PAPER -e VERSION=1.9.4 \
+        -p 25565:25565 -e EULA=TRUE --name mc itzg/minecraft-server
+
+__NOTE: to avoid pegging the CPU when running PaperSpigot,__ you will need to
+pass `--noconsole` at the very end of the command line and not use `-it`. For example,
+
+    docker run -d -v /path/on/host:/data \
+        -e TYPE=PAPER -e VERSION=1.9.4 \
+        -p 25565:25565 -e EULA=TRUE --name mc itzg/minecraft-server --noconsole
+
+You can install Bukkit plugins in two ways...
+
+### Using the /data volume
+
+This is the easiest way if you are using a persistent `/data` mount.
+
+To do this, you will need to attach the container's `/data` directory
+(see "Attaching data directory to host filesystem”).
+Then, you can add plugins to the `/path/on/host/plugins` folder you chose. From the example above,
+the `/path/on/host` folder contents look like:
+
+```
+/path/on/host
+├── plugins
+│   └── ... INSTALL PLUGINS HERE ...
+├── ops.json
+├── server.properties
+├── whitelist.json
+└── ...
+```
+
+If you add plugins while the container is running, you'll need to restart it to pick those
+up:
+
+    docker stop mc
+    docker start mc
+
+### Using separate mounts
+
+This is the easiest way if you are using an ephemeral `/data` filesystem,
+or downloading a world with the `WORLD` option.
+
+There is one additional volume that can be mounted; `/plugins`.  
+Any files in this filesystem will be copied over to the main
+`/data/plugins` filesystem before starting Minecraft.
+
+This works well if you want to have a common set of plugins in a separate
 location, but still have multiple worlds with different server requirements
 in either persistent volumes or a downloadable archive.
 
@@ -215,7 +277,7 @@ minecraft-server:
     - "25565:25565"
 
   environment:
-    EULA: TRUE
+    EULA: "TRUE"
 
   image: itzg/minecraft-server
 
@@ -266,6 +328,114 @@ downloaded, scaled, and converted from any other image format:
 
     docker run -d -e ICON=http://..../some/image.png ...
 
+### Rcon
+
+To use rcon use the `ENABLE_RCON` and `RCON_PASSORD` variables.
+By default rcon port will be `25575` but can easily be changed with the `RCON_PORT` variable.
+
+    docker run -d -e ENABLE_RCON=true -e RCON_PASSWORD=testing
+
+### Query
+
+Enabling this will enable the gamespy query protocol.
+By default the query port will be `25565` (UDP) but can easily be changed with the `QUERY_PORT` variable.
+
+    docker run -d -e ENABLE_QUERY=true
+
+
+### Max players
+
+By default max players is 20, you can increase this with the `MAX_PLAYERS` variable.
+
+    docker run -d -e MAX_PLAYERS=50
+
+
+### Max world size
+
+This sets the maximum possible size in blocks, expressed as a radius, that the world border can obtain.
+
+    docker run -d -e MAX_WORLD_SIZE=10000   
+
+### Allow Nether
+
+Allows players to travel to the Nether.
+
+    docker run -d -e ALLOW_NETHER=true
+
+### Announce Player Achievements
+
+Allows server to announce when a player gets an achievement.
+
+    docker run -d -e ANNOUNCE_PLAYER_ACHIEVEMENTS=true   
+
+### Enable  Command Block
+
+Enables command blocks
+
+     docker run -d -e ENABLE_COMMAND_BLOCK=true
+
+### Force Gamemode
+
+Force players to join in the default game mode.
+
+* false - Players will join in the gamemode they left in.
+* true - Players will always join in the default gamemode.
+
+    `docker run -d -e FORCE_GAMEMODE=false`
+
+### Generate Structures
+
+Defines whether structures (such as villages) will be generated.
+
+* false - Structures will not be generated in new chunks.
+* true - Structures will be generated in new chunks.
+
+    `docker run -d -e GENERATE_STRUCTURES=true`
+
+### Hardcore
+
+If set to true, players will be set to spectator mode if they die.
+
+    docker run -d -e HARDCORE=false
+
+### Max Build Height
+
+The maximum height in which building is allowed.
+Terrain may still naturally generate above a low height limit.
+
+    docker run -d -e MAX_BUILD_HEIGHT=256
+
+### Max Tick Time
+
+The maximum number of milliseconds a single tick may take before the server watchdog stops the server with the message, A single server tick took 60.00 seconds (should be max 0.05); Considering it to be crashed, server will forcibly shutdown. Once this criteria is met, it calls System.exit(1).
+Setting this to -1 will disable watchdog entirely
+
+    docker run -d -e MAX_TICK_TIME=60000
+
+### Spawn Animals
+
+Determines if animals will be able to spawn.
+
+    docker run -d -e SPAWN_ANIMALS=true
+
+### Spawn Monsters
+
+Determines if monsters will be spawned.
+
+    docker run -d -e SPAWN_MONSTERS=true
+
+### Spawn NPCs
+
+Determines if villagers will be spawned.
+
+    docker run -d -e SPAWN_NPCS=true
+
+### View Distance
+Sets the amount of world data the server sends the client, measured in chunks in each direction of the player (radius, not diameter).
+It determines the server-side viewing distance.
+
+    docker run -d -e VIEW_DISTANCE=10
+
 ### Level Seed
 
 If you want to create the Minecraft level with a specific seed, use `SEED`, such as
@@ -340,7 +510,7 @@ where the default is "world":
 Instead of mounting the `/data` volume, you can instead specify the URL of
 a ZIP file containing an archived world.  This will be downloaded, and
 unpacked in the `/data` directory; if it does not contain a subdirectory
-called `world/` then it will be searched for a file `level.dat` and the 
+called `world/` then it will be searched for a file `level.dat` and the
 containing subdirectory renamed to `world`.  This means that most of the
 archived Minecraft worlds downloadable from the Internet will already be in
 the correct format.
@@ -360,7 +530,7 @@ name of a linked container.
 ### Downloadable mod/plugin pack for Forge, Bukkit, and Spigot Servers
 
 Like the `WORLD` option above, you can specify the URL of a "mod pack"
-to download and install into `mods` for Forge or `plugins` for Bukkit/Spigot. 
+to download and install into `mods` for Forge or `plugins` for Bukkit/Spigot.
 To use this option pass the environment variable `MODPACK`, such as
 
     docker run -d -e MODPACK=http://www.example.com/mods/modpack.zip ...
@@ -369,7 +539,13 @@ To use this option pass the environment variable `MODPACK`, such as
 top level of the zip archive. Make sure the jars are compatible with the
 particular `TYPE` of server you are running.
 
-## JVM Configuration
+### Online mode
+
+By default, server checks connecting players against Minecraft's account database. If you want to create an offline server or your server is not connected to the internet, you can disable the server to try connecting to minecraft.net to authenticate players with environment variable `ONLINE_MODE`, like this
+
+    docker run -d -e ONLINE_MODE=FALSE ...
+
+## Miscellaneous Options
 
 ### Memory Limit
 
@@ -377,3 +553,7 @@ The Java memory limit can be adjusted using the `JVM_OPTS` environment variable,
 the setting shown in the example (max and min at 1024 MB):
 
     docker run -e 'JVM_OPTS=-Xmx1024M -Xms1024M' ...
+
+### /data ownership
+
+In order to adapt to differences in `UID` and `GID` settings the entry script will attempt to correct ownership and writability of the `/data` directory. This logic can be disabled by setting `-e SKIP_OWNERSHIP_FIX=TRUE`.

minecraft-server/docker-compose.yml

@@ -3,7 +3,7 @@ minecraft-server:
     - "25565:25565"
 
   environment:
-    EULA: TRUE
+    EULA: "TRUE"
 
   image: itzg/minecraft-server
 

minecraft-server/mcadmin.jq

@@ -0,0 +1,7 @@
+.[] |
+ select(.elements | length > 1) |
+  select(.elements[].elements[] | select(.class == "version" and .text == $version)) |
+   .elements[].elements[] | 
+    select(.class|contains("server-jar")) |
+     .elements[] | select(.name="a") |
+      .href

minecraft-server/server.properties

@@ -5,11 +5,14 @@ enable-query=false
 allow-flight=false
 announce-player-achievements=true
 server-port=25565
+rcon.port=25575
+query.port=25565
 level-type=DEFAULT
 enable-rcon=false
 force-gamemode=false
 level-seed=
 server-ip=
+max-tick-time=60000
 max-build-height=256
 spawn-npcs=true
 white-list=false
@@ -31,3 +34,5 @@ view-distance=10
 spawn-protection=16
 motd=A Minecraft Server powered by Docker
 generator-settings=
+rcon.password=
+max-world-size=29999984

minecraft-server/start-minecraft.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-umask 002
+#umask 002
+export HOME=/data
 
 if [ ! -e /data/eula.txt ]; then
   if [ "$EULA" != "" ]; then
@@ -17,101 +18,193 @@ if [ ! -e /data/eula.txt ]; then
   fi
 fi
 
+VERSIONS_JSON=https://launchermeta.mojang.com/mc/game/version_manifest.json
+
 echo "Checking version information."
 case "X$VERSION" in
   X|XLATEST|Xlatest)
-    VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'`
+    VANILLA_VERSION=`curl -sSL $VERSIONS_JSON | jq -r '.latest.release'`
   ;;
   XSNAPSHOT|Xsnapshot)
-    VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.snapshot)'`
+    VANILLA_VERSION=`curl -sSL $VERSIONS_JSON | jq -r '.latest.snapshot'`
   ;;
   X[1-9]*)
     VANILLA_VERSION=$VERSION
   ;;
   *)
-    VANILLA_VERSION=`wget -O - https://s3.amazonaws.com/Minecraft.Download/versions/versions.json | jsawk -n 'out(this.latest.release)'`
+    VANILLA_VERSION=`curl -sSL $VERSIONS_JSON | jq -r '.latest.release'`
   ;;
 esac
 
 cd /data
 
+function buildSpigotFromSource {
+  echo "Building Spigot $VANILLA_VERSION from source, might take a while, get some coffee"
+  mkdir /data/temp
+  cd /data/temp
+  wget -q -P /data/temp https://hub.spigotmc.org/jenkins/job/BuildTools/lastSuccessfulBuild/artifact/target/BuildTools.jar && \
+    java -jar /data/temp/BuildTools.jar --rev $VANILLA_VERSION 2>&1 |tee /data/spigot_build.log| while read l; do echo -n .; done; echo "done"
+  mv spigot-*.jar /data/spigot_server.jar
+  mv craftbukkit-*.jar /data/craftbukkit_server.jar
+  echo "Cleaning up"
+  rm -rf /data/temp
+  cd /data
+}
+
+function downloadSpigot {
+  local match
+  case "$TYPE" in
+    *BUKKIT|*bukkit)
+      match="Craftbukkit"
+
+      ;;
+    *)
+      match="Spigot"
+      ;;
+  esac
+
+  downloadUrl=$(restify --class=jar-div https://mcadmin.net/ | \
+    jq --arg version "$match $VANILLA_VERSION" -r -f /usr/share/mcadmin.jq)
+  if [[ -n $downloadUrl ]]; then
+    echo "Downloading $match"
+    wget -q -O $SERVER "$downloadUrl"
+    status=$?
+    if [ $status != 0 ]; then
+      echo "ERROR: failed to download from $downloadUrl due to (error code was $status)"
+      exit 3
+    fi
+  else
+    echo "ERROR: Version $VANILLA_VERSION is not supported for $TYPE"
+    echo "       Refer to https://mcadmin.net/ for supported versions"
+    exit 2
+  fi
+}
+
+function downloadPaper {
+  local build
+  case "$VERSION" in
+    latest|LATEST|1.10)
+      build="lastSuccessfulBuild";;
+    1.9.4)
+      build="773";;
+    1.9.2)
+      build="727";;
+    1.9)
+      build="612";;
+    1.8.8)
+      build="443";;
+    *)
+      build="nosupp";;
+  esac
+
+  if [ $build != "nosupp" ]; then
+    downloadUrl="https://ci.destroystokyo.com/job/PaperSpigot/$build/artifact/paperclip.jar"
+    wget -q -O $SERVER "$downloadUrl"
+    status=$?
+    if [ $status != 0 ]; then
+      echo "ERROR: failed to download from $downloadUrl due to (error code was $status)"
+      exit 3
+    fi
+  else
+    echo "ERROR: Version $VERSION is not supported for $TYPE"
+    echo "       Refer to https://ci.destroystokyo.com/job/PaperSpigot/"
+    echo "       for supported versions"
+    exit 2
+  fi
+}
+
+function installForge {
+  TYPE=FORGE
+  norm=$VANILLA_VERSION
+
+  echo "Checking Forge version information."
+  case $FORGEVERSION in
+    RECOMMENDED)
+      curl -o /tmp/forge.json -sSL http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json
+      FORGE_VERSION=$(cat /tmp/forge.json | jq -r ".promos[\"$norm-recommended\"]")
+      if [ $FORGE_VERSION = null ]; then
+        FORGE_VERSION=$(cat /tmp/forge.json | jq -r ".promos[\"$norm-latest\"]")
+        if [ $FORGE_VERSION = null ]; then
+          echo "ERROR: Version $FORGE_VERSION is not supported by Forge"
+          echo "       Refer to http://files.minecraftforge.net/ for supported versions"
+          exit 2
+        fi
+      fi
+      ;;
+
+    *)
+      FORGE_VERSION=$FORGEVERSION
+      ;;
+  esac
+
+  # URL format changed for 1.7.10 from 10.13.2.1300
+  sorted=$((echo $FORGE_VERSION; echo 10.13.2.1300) | sort -V | head -1)
+  if [[ $norm == '1.7.10' && $sorted == '10.13.2.1300' ]]; then
+      # if $FORGEVERSION >= 10.13.2.1300
+      normForgeVersion="$norm-$FORGE_VERSION-$norm"
+  else
+      normForgeVersion="$norm-$FORGE_VERSION"
+  fi
+
+  FORGE_INSTALLER="forge-$normForgeVersion-installer.jar"
+  SERVER="forge-$normForgeVersion-universal.jar"
+
+  if [ ! -e "$SERVER" ]; then
+    echo "Downloading $FORGE_INSTALLER ..."
+    wget -q http://files.minecraftforge.net/maven/net/minecraftforge/forge/$normForgeVersion/$FORGE_INSTALLER
+    echo "Installing $SERVER"
+    java -jar $FORGE_INSTALLER --installServer
+  fi
+}
+
+function installVanilla {
+  SERVER="minecraft_server.$VANILLA_VERSION.jar"
+
+  if [ ! -e $SERVER ]; then
+    echo "Downloading $SERVER ..."
+    wget -q https://s3.amazonaws.com/Minecraft.Download/versions/$VANILLA_VERSION/$SERVER
+  fi
+}
+
 echo "Checking type information."
 case "$TYPE" in
   *BUKKIT|*bukkit|SPIGOT|spigot)
-    TYPE=SPIGOT
     case "$TYPE" in
       *BUKKIT|*bukkit)
-        echo "Downloading latest CraftBukkit $VANILLA_VERSION server ..."
         SERVER=craftbukkit_server.jar
         ;;
       *)
-        echo "Downloading latest Spigot $VANILLA_VERSION server ..."
         SERVER=spigot_server.jar
         ;;
     esac
-    case $VANILLA_VERSION in
-      1.8*)
-        URL=/spigot18/$SERVER
-        ;;
-      1.9*)
-        URL=/spigot19/$SERVER
-        ;;
-      *)
-        echo "That version of $SERVER is not available."
-        exit 1
-      ;;
-    esac
 
-    #attempt https, and if it fails, fallback to http and download that way. Display error if neither works.
-    wget -q -N $SERVER https://getspigot.org$URL || \
-    	(echo "Falling back to http, unable to contact server using https..." && \
-    	wget -q -N $SERVER http://getspigot.org$URL) || \
-    	echo "Unable to download new copy of spigot server"
+    if [ ! -f $SERVER ]; then
+       if [[ "$BUILD_SPIGOT_FROM_SOURCE" = TRUE || "$BUILD_SPIGOT_FROM_SOURCE" = true || "$BUILD_FROM_SOURCE" = TRUE || "$BUILD_FROM_SOURCE" = true ]]; then
+         buildSpigotFromSource
+       else
+         downloadSpigot
+       fi
+    fi
+    # normalize on Spigot for operations below
+    TYPE=SPIGOT
+  ;;
 
-    ;;
+  PAPER|paper)
+    SERVER=paper_server.jar
+    if [ ! -f $SERVER ]; then
+      downloadPaper
+    fi
+    # normalize on Spigot for operations below
+    TYPE=SPIGOT
+  ;;
 
   FORGE|forge)
     TYPE=FORGE
-    norm=$VANILLA_VERSION
-
-    echo "Checking Forge version information."
-    case $FORGEVERSION in
-        RECOMMENDED)
-  		FORGE_VERSION=`wget -O - http://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json | jsawk -n "out(this.promos['$norm-recommended'])"`
-        ;;
-
-        *)
-  		FORGE_VERSION=$FORGEVERSION
-        ;;
-    esac
-
-    # URL format changed for 1.7.10 from 10.13.2.1300
-    sorted=$((echo $FORGE_VERSION; echo 10.13.2.1300) | sort -V | head -1)
-    if [[ $norm == '1.7.10' && $sorted == '10.13.2.1300' ]]; then
-        # if $FORGEVERSION >= 10.13.2.1300
-        normForgeVersion="$norm-$FORGE_VERSION-$norm"
-    else
-        normForgeVersion="$norm-$FORGE_VERSION"
-    fi
-
-    FORGE_INSTALLER="forge-$normForgeVersion-installer.jar"
-    SERVER="forge-$normForgeVersion-universal.jar"
-
-    if [ ! -e "$SERVER" ]; then
-      echo "Downloading $FORGE_INSTALLER ..."
-      wget -q http://files.minecraftforge.net/maven/net/minecraftforge/forge/$normForgeVersion/$FORGE_INSTALLER
-      echo "Installing $SERVER"
-      java -jar $FORGE_INSTALLER --installServer
-    fi
+    installForge
   ;;
 
   VANILLA|vanilla)
-    SERVER="minecraft_server.$VANILLA_VERSION.jar"
-
-    if [ ! -e $SERVER ]; then
-      echo "Downloading $SERVER ..."
-      wget -q https://s3.amazonaws.com/Minecraft.Download/versions/$VANILLA_VERSION/$SERVER
-    fi
+    installVanilla
   ;;
 
   *)
@@ -122,6 +215,7 @@ case "$TYPE" in
 
 esac
 
+
 # If supplied with a URL for a world, download it and unpack
 if [[ "$WORLD" ]]; then
 case "X$WORLD" in
@@ -164,10 +258,10 @@ case "X$MODPACK" in
     wget -q -O /tmp/modpack.zip "$MODPACK"
     if [ "$TYPE" = "SPIGOT" ]; then
       mkdir -p /data/plugins
-      unzip -d /data/plugins /tmp/modpack.zip
+      unzip -o -d /data/plugins /tmp/modpack.zip
     else
       mkdir -p /data/mods
-      unzip -d /data/mods /tmp/modpack.zip
+      unzip -o -d /data/mods /tmp/modpack.zip
     fi
     rm -f /tmp/modpack.zip
     ;;
@@ -177,33 +271,57 @@ case "X$MODPACK" in
 esac
 fi
 
+function setServerProp {
+  local prop=$1
+  local var=$2
+  if [ -n "$var" ]; then
+    echo "Setting $prop to $var"
+    sed -i "/$prop\s*=/ c $prop=$var" /data/server.properties
+  fi
+
+}
+
 if [ ! -e server.properties ]; then
+  echo "Creating server.properties"
   cp /tmp/server.properties .
 
   if [ -n "$WHITELIST" ]; then
+    echo "Creating whitelist"
     sed -i "/whitelist\s*=/ c whitelist=true" /data/server.properties
     sed -i "/white-list\s*=/ c white-list=true" /data/server.properties
   fi
 
-  if [ -n "$MOTD" ]; then
-    sed -i "/motd\s*=/ c motd=$MOTD" /data/server.properties
-  fi
-
-  if [ -n "$LEVEL" ]; then
-    sed -i "/level-name\s*=/ c level-name=$LEVEL" /data/server.properties
-  fi
-
-  if [ -n "$SEED" ]; then
-    sed -i "/level-seed\s*=/ c level-seed=$SEED" /data/server.properties
-  fi
-
-  if [ -n "$PVP" ]; then
-    sed -i "/pvp\s*=/ c pvp=$PVP" /data/server.properties
-  fi
+  setServerProp "motd" "$MOTD"
+  setServerProp "allow-nether" "$ALLOW_NETHER"
+  setServerProp "announce-player-achievements" "$ANNOUNCE_PLAYER_ACHIEVEMENTS"
+  setServerProp "enable-command-block" "$ENABLE_COMMAND_BLOCK"
+  setServerProp "spawn-animals" "$SPAWN_ANIMAILS"
+  setServerProp "spawn-monsters" "$SPAWN_MONSTERS"
+  setServerProp "spawn-npcs" "$SPAWN_NPCS"
+  setServerProp "generate-structures" "$GENERATE_STRUCTURES"
+  setServerProp "spawn-npcs" "$SPAWN_NPCS"
+  setServerProp "view-distance" "$VIEW_DISTANCE"
+  setServerProp "hardcore" "$HARDCORE"
+  setServerProp "max-build-height" "$MAX_BUILD_HEIGHT"
+  setServerProp "force-gamemode" "$FORCE_GAMEMODE"
+  setServerProp "hardmax-tick-timecore" "$MAX_TICK_TIME"
+  setServerProp "enable-query" "$ENABLE_QUERY"
+  setServerProp "query.port" "$QUERY_PORT"
+  setServerProp "enable-rcon" "$ENABLE_RCON"
+  setServerProp "rcon.password" "$RCON_PASSWORD"
+  setServerProp "rcon.port" "$RCON_PORT"
+  setServerProp "max-players" "$MAX_PLAYERS"
+  setServerProp "max-world-size" "$MAX_WORLD_SIZE"
+  setServerProp "level-name" "$LEVEL"
+  setServerProp "level-seed" "$SEED"
+  setServerProp "pvp" "$PVP"
+  setServerProp "generator-settings" "$GENERATOR_SETTINGS"
+  setServerProp "online-mode" "$ONLINE_MODE"
 
   if [ -n "$LEVEL_TYPE" ]; then
     # normalize to uppercase
     LEVEL_TYPE=${LEVEL_TYPE^^}
+    echo "Setting level type to $LEVEL_TYPE"
     # check for valid values and only then set
     case $LEVEL_TYPE in
       DEFAULT|FLAT|LARGEBIOMES|AMPLIFIED|CUSTOMIZED)
@@ -216,10 +334,6 @@ if [ ! -e server.properties ]; then
     esac
   fi
 
-  if [ -n "$GENERATOR_SETTINGS" ]; then
-    sed -i "/generator-settings\s*=/ c generator-settings=$GENERATOR_SETTINGS" /data/server.properties
-  fi
-
   if [ -n "$DIFFICULTY" ]; then
     case $DIFFICULTY in
       peaceful|0)
@@ -239,14 +353,16 @@ if [ ! -e server.properties ]; then
         exit 1
         ;;
     esac
+    echo "Setting difficulty to $DIFFICULTY"
     sed -i "/difficulty\s*=/ c difficulty=$DIFFICULTY" /data/server.properties
   fi
 
   if [ -n "$MODE" ]; then
+    echo "Setting mode"
     case ${MODE,,?} in
       0|1|2|3)
         ;;
-      s*)
+      su*)
         MODE=0
         ;;
       c*)
@@ -255,7 +371,7 @@ if [ ! -e server.properties ]; then
       a*)
         MODE=2
         ;;
-      s*)
+      sp*)
         MODE=3
         ;;
       *)
@@ -270,10 +386,12 @@ fi
 
 
 if [ -n "$OPS" -a ! -e ops.txt.converted ]; then
+  echo "Setting ops"
   echo $OPS | awk -v RS=, '{print}' >> ops.txt
 fi
 
 if [ -n "$WHITELIST" -a ! -e white-list.txt.converted ]; then
+  echo "Setting whitelist"
   echo $WHITELIST | awk -v RS=, '{print}' >> white-list.txt
 fi
 
@@ -317,18 +435,22 @@ do
 done
 
 if [ "$TYPE" = "SPIGOT" ]; then
-  echo Copying any Bukkit plugins over
   if [ -d /plugins ]; then
+    echo Copying any Bukkit plugins over
     cp -r /plugins /data
   fi
 fi
 
+if [[ $CONSOLE = false ]]; then
+  EXTRA_ARGS=--noconsole
+else
+  EXTRA_ARGS=""
+fi
+
 # If we have a bootstrap.txt file... feed that in to the server stdin
 if [ -f /data/bootstrap.txt ];
 then
-    exec java $JVM_OPTS -jar $SERVER < /data/bootstrap.txt
+    exec java $JVM_OPTS -jar $SERVER "$@" $EXTRA_ARGS < /data/bootstrap.txt
 else
-    exec java $JVM_OPTS -jar $SERVER
+    exec java $JVM_OPTS -jar $SERVER "$@" $EXTRA_ARGS
 fi
-
-exec java $JVM_OPTS -jar $SERVER

minecraft-server/start.sh

@@ -4,12 +4,19 @@ set -e
 usermod --uid $UID minecraft
 groupmod --gid $GID minecraft
 
-chown -R minecraft:minecraft /data /start-minecraft
-chmod -R g+wX /data /start-minecraft
+if [ "$SKIP_OWNERSHIP_FIX" != "TRUE" ]; then
+  fix_ownership() {
+    dir=$1
+    if ! sudo -u minecraft test -w $dir; then
+      echo "Correcting writability of $dir ..."
+      chown -R minecraft:minecraft $dir
+      chmod -R u+w $dir
+    fi
+  }
 
-while lsof -- /start-minecraft; do
-  echo -n "."
-  sleep 1
-done
-echo "...switching to user 'minecraft'"
-exec sudo -E -u minecraft /start-minecraft
+  fix_ownership /data
+  fix_ownership /home/minecraft
+fi
+
+echo "Switching to user 'minecraft'"
+exec sudo -E -u minecraft /start-minecraft "$@"

titan-gremlin/README.md

@@ -17,11 +17,11 @@ the next time you (re)start your container.
 
 # Connecting to Cassandra and Elasticsearch Containers
 
-First start containers for Cassandra and Elasticsearch, where the `--name` you choose
-can be arbitrary or left off to use a generated name.
+First start containers for Cassandra and Elasticsearch (pre-2.x),
+where the `--name` you choose can be arbitrary or left off to use a generated name.
 
-    docker run -d --name gremlin-cass itzg/cassandra 
-    docker run -d --name gremlin-es itzg/elasticsearch
+    docker run -d --name gremlin-cass itzg/cassandra
+    docker run -d --name gremlin-es itzg/elasticsearch:1.x
 
 Now start Gremlin with networking links to those containers with the aliases
 
@@ -43,3 +43,21 @@ and with that you can follow the
     ==>age=10000
     gremlin> saturn.in('father').in('father').name
     ==>hercules
+
+# Running and Connecting with Docker Compose
+
+Create the following Compose content as the file `docker-compose.yml` and in
+that directory invoke `docker-compose run titan` to run the Gremlin shell
+with the supporting Elasticsearch and Cassandra containers.
+
+```
+titan:
+    image: itzg/titan-gremlin
+    links:
+        - cass
+        - es
+cass:
+    image: itzg/cassandra
+es:
+    image: itzg/elasticsearch:1.x
+```

titan-gremlin/start-gremlin.sh

@@ -48,7 +48,7 @@ END
 index.search.hostname=$esAddr
 END
   fi
-  
+
 fi
 
 if [ -n "$shortcut" ]; then
@@ -59,4 +59,6 @@ END
   args="$args /tmp/init.groovy"
 fi
 
+# Allow a little extra time for Cassandra to be ready
+sleep 1
 exec $TITAN_HOME/bin/gremlin.sh $args