Merge branch 'v2016.1'

* v2016.1: (24 commits)
  Create sha512sum.txt instead of sha256sum.txt
  Fix site prefix
  Fix error when arch dir exists
  Add verbose flag to make
  Continue if a target fails, merge builds in output directory, clean up
  Build with BROKEN=1 by default, fix error creating manifest
  Fix unneeded check parameter
  Add timing for build.sh
  Use python instead of shell for build script, only build some targets by default
  Add cleanup between builds, generate manifest
  The file targets.mk now doesn't stop the build any more
  Fix update path
  Use downloads.openwrt.org for packages
  Clean up, prepare stable release 0.8.0
  Update site.conf
  Added public signing key for Torben
  Added public signing key for Till
  Update image URLs
  Add pubkey for markus
  use dirclean instead of clean between builds
  ...

Dockerfile

@@ -1,17 +1,21 @@
 FROM debian:jessie
 MAINTAINER Markus Lindenberg <markus@lindenberg.io>
 
-ENV GLUON_TAG v2015.1.2
+ENV GLUON_SITE ffdo
+ENV GLUON_TAG v2016.1
+ENV GLUON_RELEASE 0.8.0
+ENV GLUON_BRANCH stable
+ENV GLUON_BROKEN 1
+ENV GLUON_TARGETS ar71xx-generic ar71xx-nand mpc85xx-generic x86-generic x86-64
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBIAN_PRIORITY critical
 ENV DEBCONF_NOWARNINGS yes
 
 RUN apt-get update
-RUN apt-get -y install --no-install-recommends ca-certificates python wget file git subversion build-essential gawk unzip libncurses5-dev zlib1g-dev && apt-get clean
+RUN apt-get -y install --no-install-recommends ca-certificates python python3 wget file git subversion build-essential gawk unzip libncurses5-dev zlib1g-dev openssl libssl-dev && apt-get clean
 
-
-ADD docker/build.sh /usr/src/build.sh
+ADD docker-build.py /usr/src/build.py
 ADD site.mk /usr/src/site.mk
 ADD site.conf /usr/src/site.conf
 ADD i18n /usr/src/i18n
@@ -20,6 +24,7 @@ RUN adduser --system --home /usr/src/build build
 USER build
 WORKDIR /usr/src/build
 RUN git config --global user.email "technik@freifunk-dortmund.de"
-RUN git config --global user.name "FFDO Gluon Build System"
+RUN git config --global user.name "FFDO Gluon Build Container"
+
+CMD ["/usr/src/build.py"]
 
-CMD ["/usr/src/build.sh"]

README.md

@@ -5,8 +5,15 @@ FF Dortmund (FFDO) specific Gluon configuration
 
 See https://docs.docker.com/installation/#installation on how to get Docker.
 
+
+```
+./build.sh
+```
+
+## Cleaning up
+
 ```
-docker build -t ffdobuild .
-docker run ffdobuild
+docker rm ffdobuild
+docker rmi ffdobuild
 ```
-After a successful build you can remove the used container. Use `docker ps -a` to find the container ID and `docker rm <container>` to remove the container. Use `docker rmi ffdobuild` to remove the image used to create the container.
+

build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker rm -f ffdobuild
+docker build -t ffdobuild .
+time docker run --name ffdobuild ffdobuild

docker-build.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+from os import environ, makedirs, chdir, listdir, rename
+from os.path import isdir
+from subprocess import call, check_call
+from shutil import *
+from multiprocessing import cpu_count
+from datetime import datetime
+from sys import stdout
+
+site = environ['GLUON_SITE']
+release = environ['GLUON_RELEASE']
+branch = environ['GLUON_BRANCH']
+broken = environ['GLUON_BROKEN']
+
+home = '/usr/src/build'
+
+gluondir = '%s/gluon' % home
+
+outdir = '%s/output/%s' % (home, release)
+logdir = '%s/log' % outdir
+
+siteconf = '/usr/src/site.conf'
+sitemk = '/usr/src/site.mk'
+i18ndir = '/usr/src/i18n'
+
+factorydir = '%s/images/factory' % outdir
+sysupdir = '%s/images/sysupgrade' % outdir
+modulesdir = '%s/modules' % outdir
+
+makedirs(factorydir)
+makedirs(sysupdir)
+makedirs(modulesdir)
+makedirs(logdir)
+
+start = datetime.now()
+
+def format_duration(d):
+    s = d.total_seconds()
+    return '%.2dh%.2dm%06.3fs' % (s//3600, (s//60)%60, s%60)
+
+# Clone Gluon
+if isdir(gluondir):
+    rmtree(gluondir)
+
+print('Cloning Gluon... ', end=''); stdout.flush()
+with open('%s/git.log' % logdir, 'w') as log:
+    check_call('git clone https://github.com/freifunk-gluon/gluon.git %s -b "%s"' % (gluondir, environ['GLUON_TAG']), stdout=log, stderr=log, shell=True)
+print('OK'); stdout.flush()
+
+# Add site configuration
+sitedir = '%s/site' % gluondir
+makedirs(sitedir)
+copy(siteconf, sitedir)
+copy(sitemk, sitedir)
+copytree(i18ndir, '%s/i18n' % sitedir)
+
+# Prepare
+chdir(gluondir)
+print('Updating other repositories... ', end=''); stdout.flush()
+with open('%s/update.log' % logdir, 'w') as log:
+    check_call('make update V=s', stdout=log, stderr=log, shell=True)
+print('OK'); stdout.flush()
+
+# Choose targets to build
+if 'GLUON_TARGETS' in environ:
+    targets = environ['GLUON_TARGETS'].split()
+else:
+    targets = [f for f in listdir('targets') if isdir('targets/%s' % f)]
+
+# Build
+for target in targets:
+    print('Building for target %s... ' % target, end=''); stdout.flush()
+    arch, variant = target.split('-')
+    target_start = datetime.now()
+
+    with open('%s/%s_stdout.log' % (logdir, target), 'w') as logout, open('%s/%s_stderr.log' % (logdir, target), 'w') as logerr:
+        rc = call('make -j %s GLUON_BRANCH=%s BROKEN=%s GLUON_TARGET=%s V=s' % (cpu_count()+1, branch, broken, target), stdout=logout, stderr=logerr, shell=True)
+    duration = format_duration(datetime.now() - target_start)
+    if rc == 0:
+        print('OK in', duration)
+
+        # Create manifest
+        print('Creating manifest... ', end=''); stdout.flush()
+        with open('%s/%s_manifest.log' % (logdir, target), 'w') as log:
+            rc = call('make manifest GLUON_BRANCH=%s BROKEN=%s GLUON_TARGET=%s V=s' % (branch, broken, target), stdout=log, stderr=log, shell=True)
+        if rc == 0:
+            rename('output/images/sysupgrade/%s.manifest' % branch, 'output/images/sysupgrade/%s.%s.manifest' % (target, branch))
+            print('OK')
+        else:
+            print('FAILED')
+        stdout.flush()
+
+        # Move images to output
+        for f in listdir('output/images/factory'):
+            rename('output/images/factory/%s' % f, '%s/%s' % (factorydir, f))
+        for f in listdir('output/images/sysupgrade'):
+            rename('output/images/sysupgrade/%s' % f, '%s/%s' % (sysupdir, f))
+
+        # Move modules to output
+        try:
+            makedirs('%s/%s' % (modulesdir, arch))
+        except FileExistsError:
+            pass
+        variantdir = '%s/%s/%s' % (modulesdir, arch, variant)
+        rename('output/modules/gluon-%s-%s/%s/%s' % (site, release, arch, variant), variantdir)
+
+        # Checksum modules
+        print('Creating SHA512 sums for modules... ', end=''); stdout.flush()
+        chdir(variantdir)
+        check_call('sha512sum * > sha512sum.txt', shell=True)
+        chdir(gluondir)
+        print('OK')
+    else:
+        print('FAILED after', duration)
+
+    # Clean up
+    print('Cleaning up...', end=''); stdout.flush()
+    with open('%s/%s_cleanup.log' % (logdir, target), 'w') as log:
+        check_call('make dirclean V=s', stdout=log, stderr=log, shell=True)
+    print('OK'); stdout.flush()
+
+print('Creating SHA512 sums for images... ', end=''); stdout.flush()
+for d in (factorydir, sysupdir):
+    chdir(d)
+    check_call('sha512sum * > sha512sum.txt', shell=True)
+print('OK')
+
+print('''
+BUILD FINISHED in %s
+
+You can copy the resulting images from the container using:
+docker cp %s:/usr/src/build/output <destination>
+'''% (format_duration(datetime.now() - start), environ['HOSTNAME']))
+

docker/build.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# Clean up
-rm -rf gluon
-git clone https://github.com/freifunk-gluon/gluon.git gluon -b "${GLUON_TAG}"
-
-# Add site configuration
-mkdir -p gluon/site
-cp /usr/src/site.mk gluon/site/
-cp /usr/src/site.conf gluon/site/
-cp -r /usr/src/i18n gluon/site/
-
-# Build
-cd gluon
-make update
-
-for target in $(ls targets/)
-do
-	if [ -d "targets/$target" ]; then
-		echo "Building for target $target"
-		time make -j $(($(nproc)+1)) BROKEN=1 GLUON_TARGET=$target
-	fi
-done
-
-set +x
-echo -e "\nBUILD FINISHED\n"
-echo "You can copy the resulting images from the container using:"
-echo -e "\ndocker cp ${HOSTNAME}:/usr/src/build/gluon/images <destination>\n"

i18n/de.po

@@ -19,9 +19,16 @@ msgstr ""
 
 msgid "gluon-config-mode:pubkey"
 msgstr ""
+"<p>"
 "Dies ist der &ouml;ffentliche Schl&uuml;ssel deines Freifunk-Knotens zur Information."
 "Der Knoten ist nun direkt nutzbar."
 "Bitte vergiss nicht das Netzwerkkabel vom gelben LAN Port in den blauen WAN Port umzustecken."
+"</p>"
+"<div class=\"the-key\">"
+" # <%= hostname %>"
+" <br/>"
+"<%= pubkey %>"
+"</div>"
 
 msgid "gluon-config-mode:reboot"
 msgstr ""

i18n/en.po

@@ -17,7 +17,14 @@ msgstr ""
 
 msgid "gluon-config-mode:pubkey"
 msgstr ""
+"<p>"
 "This is your Freifunk node's public key. You can use your node right away."
+"</p>"
+"<div class=\"the-key\">"
+" # <%= hostname %>"
+" <br/>"
+"<%= pubkey %>"
+"</div>"
 
 msgid "gluon-config-mode:reboot"
 msgstr ""

site.conf

@@ -1,31 +1,44 @@
 {
-	hostname_prefix = 'FF-DO',
+	hostname_prefix = 'FF-DO-',
 	site_name = 'Freifunk Dortmund',
 	site_code = 'ffdo',
-	opkg_repo = 'http://openwrt.draic.info/barrier_breaker/14.07/%S/packages',
 
 	prefix4 = '10.233.0.0/16',
 	prefix6 = '2a03:2260:50:5::/64',
 
 	timezone = 'CET-1CEST,M3.5.0,M10.5.0/3', -- Europe/Berlin
-	ntp_servers = {'1.ntp.services.ffdo.de','2.ntp.services.ffdo.de','3.ntp.services.ffdo.de', '4.ntp.services.ffdo.de'},
+	ntp_servers = {'ntp.services.ffdo.de'},
 	regdom = 'DE',
 
+	opkg = {
+		openwrt = 'http://downloads.openwrt.org/%n/%v/%S/packages',
+		extra = {
+			modules = 'http://update.services.ffdo.de/%GS/releases/%GR/modules/%S',
+		},
+	},
+
 	wifi24 = {
-		ssid = 'Freifunk',
 		channel = 1,
-		htmode = 'HT40+',
-		mesh_ssid = 'wifimesh-dortmund',
-		mesh_bssid = '02:ff:d0:09:ff:01',
-		mesh_mcast_rate = 12000,
+		ap = {
+			ssid = 'Freifunk',
+		},
+		ibss = {
+			ssid = 'wifimesh-dortmund',
+			bssid = '02:ff:d0:09:ff:01',
+			mcast_rate = 12000,
+		},
 	},
+
 	wifi5 = {
-		ssid = 'Freifunk (5GHz)',
 		channel = 44,
-		htmode = 'HT40+',
-		mesh_ssid = 'wifimesh-dortmund5',
-		mesh_bssid = '02:ff:d0:09:ff:02',
-		mesh_mcast_rate = 12000,
+		ap = {
+			ssid = 'Freifunk (5GHz)',
+		},
+		ibss = {
+			ssid = 'wifimesh-dortmund5',
+			bssid = '02:ff:d0:09:ff:02',
+			mcast_rate = 12000,
+		},
 	},
 
 	next_node = {
@@ -38,7 +51,7 @@
 		methods = {'salsa2012+umac'},
 		mtu = 1280,
 		groups = {
-			do01 = {
+			ffdo = {
 				limit = 1,
 				peers = {
 					do01100 = {
@@ -49,11 +62,6 @@
 						key = 'd2de261203853d14389d80ca004c98d6a67c244bfe4096b7a6f36a0a8484c55a',
 						remotes = {'"snode01-2.ffdo.de" port 10000'},
 					},
-				},
-			},
-			do02 = {
-				limit = 1,
-				peers = {
 					do02100 = {
 						key = '12b94258d16f864c863f7bab07a77514eddffe3ba01c9d4948bd84a192d28f42',
 						remotes = {'"snode02-1.ffdo.de" port 10000'},
@@ -64,11 +72,24 @@
 					},
 				},
 			},
+			do01 = {
+				limit = 0,
+				peers = {},
+			},
+			do02 = {
+				limit = 0,
+				peers = {},
+			},
 			backbone = {
 				limit = 0,
 				peers = {},
 			},
 		},
+		bandwidth_limit = {
+			enabled = false,
+			egress = 4000,
+			ingress = 30000,
+		},
 	},
 
 	autoupdater = {
@@ -76,43 +97,40 @@
 		branches = {
 			stable = {
 				name = 'stable',
-				mirrors = {'http://update.services.ffdo.de/stable/sysupgrade'},
+				mirrors = {'http://update.services.ffdo.de/ffdo/stable/images/sysupgrade'},
 				probability = 0.08,
-				good_signatures = 1,
+				good_signatures = 2,
 				pubkeys = {
-					'f64c53b9f592335bbbbad1ce44836510a2e60bd445be8bd8139832c7dae0101d', -- Chris
-					'4bcf080d3937310ea3f5ee3678bff5c839679b69c8b2529ba1371b710dd046b6', -- Philip
+					'4b74a95381fdb8d4db0f69effde000befe8e58a14996a8f732213cc40627b7fe', -- Markus
+					'64825ce2492cf99c51abcba32bd08bd8f880da58e606b3957159c78725d739ca', -- Till
+					'497223af7ce512a31d7c282765951063d781f46f6e6a890faa3bf2e03fe2e19c', -- Torben
+					'6274bc371973f6cedd6c63523388fda3a938df26a1cfbe91378c8c0a1bf6efca', -- Tim
 				},
 			},
 			beta = {
 				name = 'beta',
-				mirrors = {'http://update.services.ffdo.de/beta/sysupgrade'},
+				mirrors = {'http://update.services.ffdo.de/ffdo/beta/images/sysupgrade'},
 				probability = 0.08,
-				good_signatures = 1,
+				good_signatures = 2,
 				pubkeys = {
-					'f64c53b9f592335bbbbad1ce44836510a2e60bd445be8bd8139832c7dae0101d', -- Chris
-					'4bcf080d3937310ea3f5ee3678bff5c839679b69c8b2529ba1371b710dd046b6', -- Philip
+					'4b74a95381fdb8d4db0f69effde000befe8e58a14996a8f732213cc40627b7fe', -- Markus
+					'64825ce2492cf99c51abcba32bd08bd8f880da58e606b3957159c78725d739ca', -- Till
+					'497223af7ce512a31d7c282765951063d781f46f6e6a890faa3bf2e03fe2e19c', -- Torben
+					'6274bc371973f6cedd6c63523388fda3a938df26a1cfbe91378c8c0a1bf6efca', -- Tim
 				},
 			},
 			experimental = {
 				name = 'experimental',
-				mirrors = {'http://update.services.ffdo.de/experimental/sysupgrade'},
+				mirrors = {'http://update.services.ffdo.de/ffdo/experimental/images/sysupgrade'},
 				probability = 0.08,
 				good_signatures = 1,
 				pubkeys = {
-					'f64c53b9f592335bbbbad1ce44836510a2e60bd445be8bd8139832c7dae0101d', -- Chris
-					'4bcf080d3937310ea3f5ee3678bff5c839679b69c8b2529ba1371b710dd046b6', -- Philip
+					'4b74a95381fdb8d4db0f69effde000befe8e58a14996a8f732213cc40627b7fe', -- Markus
+					'64825ce2492cf99c51abcba32bd08bd8f880da58e606b3957159c78725d739ca', -- Till
+					'497223af7ce512a31d7c282765951063d781f46f6e6a890faa3bf2e03fe2e19c', -- Torben
+					'6274bc371973f6cedd6c63523388fda3a938df26a1cfbe91378c8c0a1bf6efca', -- Tim
 				},
 			},
 		},
 	},
-
-	simple_tc = {
-		mesh_vpn = {
-			ifname = 'mesh-vpn',
-			enabled = false,
-			limit_egress = 4000,
-			limit_ingress = 30000,
-		},
-	},
 }

site.mk

@@ -1,41 +1,55 @@
+##	GLUON_SITE_PACKAGES
+#		specify gluon/openwrt packages to include here
+#		The gluon-mesh-batman-adv-* package must come first because of the dependency resolution
+
 GLUON_SITE_PACKAGES := \
-        gluon-mesh-batman-adv-15 \
-        gluon-alfred \
-        gluon-announced \
-        gluon-autoupdater \
-        gluon-setup-mode \
-        gluon-config-mode-core \
-        gluon-config-mode-autoupdater \
-        gluon-config-mode-hostname \
-        gluon-config-mode-mesh-vpn \
-        gluon-config-mode-geo-location \
-        gluon-config-mode-contact-info \
-        gluon-ebtables-filter-multicast \
-        gluon-ebtables-filter-ra-dhcp \
-        gluon-luci-admin \
-        gluon-luci-autoupdater \
-        gluon-luci-portconfig \
-        gluon-luci-private-wifi \
-        gluon-luci-wifi-config \
-        gluon-next-node \
-        gluon-mesh-vpn-fastd \
-        gluon-radvd \
-        gluon-status-page \
-        iwinfo \
-        iptables \
-        haveged
-
-
-DEFAULT_GLUON_RELEASE := 0.7.2
+	gluon-mesh-batman-adv-15 \
+	gluon-alfred \
+	gluon-respondd \
+	gluon-autoupdater \
+	gluon-config-mode-autoupdater \
+	gluon-config-mode-contact-info \
+	gluon-config-mode-core \
+	gluon-config-mode-geo-location \
+	gluon-config-mode-hostname \
+	gluon-config-mode-mesh-vpn \
+	gluon-ebtables-filter-multicast \
+	gluon-ebtables-filter-ra-dhcp \
+	gluon-luci-admin \
+	gluon-luci-autoupdater \
+	gluon-luci-portconfig \
+	gluon-luci-private-wifi \
+	gluon-luci-wifi-config \
+	gluon-next-node \
+	gluon-mesh-vpn-fastd \
+	gluon-radvd \
+	gluon-setup-mode \
+	gluon-status-page \
+	haveged \
+	iptables \
+	iwinfo
+
+##	DEFAULT_GLUON_RELEASE
+#		version string to use for images
+#		gluon relies on
+#			opkg compare-versions "$1" '>>' "$2"
+#		to decide if a version is newer or not.
+
+DEFAULT_GLUON_RELEASE := 0.0+exp$(shell date '+%Y%m%d')
+
+
+##	GLUON_RELEASE
+#		call make with custom GLUON_RELEASE flag, to use your own release version scheme.
+#		e.g.:
+#			$ make images GLUON_RELEASE=23.42+5
+#		would generate images named like this:
+#			gluon-ff%site_code%-23.42+5-%router_model%.bin
 
 # Allow overriding the release number from the command line
 GLUON_RELEASE ?= $(DEFAULT_GLUON_RELEASE)
 
+# Default priority for updates
 GLUON_PRIORITY ?= 0
-GLUON_BRANCH ?= stable
-export GLUON_BRANCH
-
-GLUON_TARGET ?= ar71xx-generic
-export GLUON_TARGET
 
+# Languages to include
 GLUON_LANGS ?= en de