From 61f3176024ea33224a9cfa560c8f9410738bfbd2 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 13 Oct 2015 10:32:14 +0200
Subject: [PATCH 01/70] debian: Switch to dh

Switch to dh to easily rebuild autotools files.
---
 debian/rules | 81 +++++++---------------------------------------------
 1 file changed, 10 insertions(+), 71 deletions(-)

diff --git a/debian/rules b/debian/rules
index 5c6f51bf..dc11016d 100755
--- a/debian/rules
+++ b/debian/rules
@@ -3,85 +3,29 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
-include /usr/share/dpkg/architecture.mk
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 
-# These are used for cross-compiling and for saving the configure script
-# from having to guess our platform (since we know it already)
-ifeq ($(DEB_HOST_GNU_TYPE),$(DEB_BUILD_GNU_TYPE))
-BUILD_SYSTEM = --build $(DEB_BUILD_GNU_TYPE)
-else
-BUILD_SYSTEM = --build $(DEB_BUILD_GNU_TYPE) --host $(DEB_HOST_GNU_TYPE)
-endif
-
-CFLAGS := $(shell DEB_BUILD_MAINT_OPTIONS=hardening=+all dpkg-buildflags --get CFLAGS)
-CPPFLAGS := $(shell DEB_BUILD_MAINT_OPTIONS=hardening=+all dpkg-buildflags --get CPPFLAGS)
-LDFLAGS := $(shell DEB_BUILD_MAINT_OPTIONS=hardening=+all dpkg-buildflags --get LDFLAGS)
-
-CFLAGS += -Wall -Wextra -Wmissing-prototypes -Wstrict-prototypes -Wlogical-op -Wshadow
-LDFLAGS += -Wl,-z,defs
 ifneq (,$(filter reprepro-nolibarchive,$(DEB_BUILD_OPTIONS)))
 	ARCHIVEFLAGS= --without-libarchive
 else
 	ARCHIVEFLAGS= --with-libarchive
 endif
-ifneq (,$(filter parallel=%,$(DEB_BUILD_OPTIONS)))
-	MAKEFLAGS += -j$(patsubst parallel=%,%,$(filter parallel=%,$(DEB_BUILD_OPTIONS)))
-endif
-
-config.status: configure
-	dh_testdir
-	./configure $(BUILD_SYSTEM) \
-		    --prefix=/usr --mandir=\$${prefix}/share/man \
-		    --with-liblzma --with-libgpgme \
-		    --with-libbz2 $(ARCHIVEFLAGS) \
-		    --disable-dependency-tracking \
-		    CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' \
-	|| ( echo configure failed with $$? ; echo BEGIN config.log: ; \
-	     cat config.log ; echo "END config.log" ; exit 1 )
 
-build-indep: build-indep-stamp
-build-arch: build-arch-stamp
-build: build-arch-stamp build-indep-stamp
+%:
+	dh $@
 
-build-indep-stamp:
-	touch build-indep-stamp
-build-arch-stamp:  config.status
-	dh_testdir
-	$(MAKE)
-	touch build-arch-stamp
+override_dh_auto_configure:
+	dh_auto_configure -- --with-libbz2 --with-liblzma --with-libgpgme $(ARCHIVEFLAGS)
 
-clean:
-	dh_testdir
-	dh_testroot
-	rm -f build-arch-stamp build-indep-stamp
-
-	# clean up after the build process.
-	if [ -e config.status ] ; then $(MAKE) distclean ; fi
-	! test -f config.log
-	dh_clean
-
-# Build architecture-independent files here.
-binary-indep: build-indep
-# We have nothing to do.
-
-# Build architecture-dependent files here.
-binary-arch: build-arch
-	dh_testdir
-	dh_testroot
-	dh_prep
-	dh_installdirs
+override_dh_auto_install:
 	$(MAKE) install DESTDIR=$(CURDIR)/debian/reprepro
 	install -D -m 644 docs/reprepro.bash_completion debian/reprepro/usr/share/bash-completion/completions/reprepro
 	install -D -m 644 docs/reprepro.zsh_completion debian/reprepro/usr/share/zsh/vendor-completions/_reprepro
+
+override_dh_installchangelogs:
 	dh_installchangelogs ChangeLog
-	dh_installdocs
-	dh_installexamples
-	dh_link
-	dh_strip
-	dh_compress
-	dh_fixperms
-	dh_installdeb
-	dh_shlibdeps
+
+override_dh_gencontrol:
 	grep -v '^reprepro:.*=' debian/reprepro.substvars > debian/reprepro.substvars.new
 	mv debian/reprepro.substvars.new debian/reprepro.substvars
 #	# if compile without libarchive, we need the program ar from binutils available
@@ -98,8 +42,3 @@ binary-arch: build-arch
 		echo "misc:Depends=" >> debian/reprepro.substvars ; \
 	fi
 	dh_gencontrol
-	dh_md5sums
-	dh_builddeb
-
-binary: binary-indep binary-arch
-.PHONY: build build-arch build-indep clean binary-indep binary-arch binary

From 1adb413bb8fd9730cbb61cad73df5a7d21a63fbb Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 27 Feb 2017 11:23:58 +0100
Subject: [PATCH 02/70] Run shunit2 tests on build time

---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index eb750151..4c2fca62 100644
--- a/debian/control
+++ b/debian/control
@@ -2,7 +2,7 @@ Source: reprepro
 Section: utils
 Priority: optional
 Maintainer: Bernhard R. Link <brlink@debian.org>
-Build-Depends: debhelper (>= 10), libgpgme-dev, libdb-dev, libz-dev, libbz2-dev, liblzma-dev, libarchive-dev
+Build-Depends: debhelper (>= 10), libgpgme-dev, libdb-dev, libz-dev, libbz2-dev, liblzma-dev, libarchive-dev, shunit2, db-util
 Standards-Version: 4.3.0
 Vcs-Browser: https://salsa.debian.org/brlink/reprepro
 Vcs-Git: https://salsa.debian.org/brlink/reprepro.git -b debian

From d55bfe6707c237584c107d9dd8f0817948ea9655 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 17:49:07 +0100
Subject: [PATCH 03/70] [testsuite] Use the host architecture by default

To be able to build packages for the tests, use the host architecture
(to avoid requiring a cross-compiler).
---
 tests/genpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/genpackage.sh b/tests/genpackage.sh
index ac8ffeca..8a971b7e 100755
--- a/tests/genpackage.sh
+++ b/tests/genpackage.sh
@@ -19,7 +19,7 @@ Maintainer: me <guess@who>
 Standards-Version: 0.0
 
 Package: $PACKAGE
-Architecture: abacus
+Architecture: ${ARCH:-$(dpkg-architecture -qDEB_HOST_ARCH)}
 Description: bla
  blub
 

From 29575bc87d9e8fd69634b506d8de68717df07ee9 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 28 Mar 2017 15:10:10 +0200
Subject: [PATCH 04/70] [testsuite] Use existing priority

Using the non-existing priority 'superfluous' causes warning messages.
---
 tests/genpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/genpackage.sh b/tests/genpackage.sh
index 8a971b7e..b97baea5 100755
--- a/tests/genpackage.sh
+++ b/tests/genpackage.sh
@@ -14,7 +14,7 @@ mkdir "$DIR"/debian
 cat >"$DIR"/debian/control <<END
 Source: $PACKAGE
 Section: $SECTION
-Priority: superfluous
+Priority: optional
 Maintainer: me <guess@who>
 Standards-Version: 0.0
 

From c1990d96f6d67a6e0e5f925819a8d8280f37d091 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 17:47:38 +0100
Subject: [PATCH 05/70] [testsuite] Use dpkg-source format 3.0

To test the handling of upstream tarballs, switch from source format 1.0
to either 3.0 (quilt) or 3.0 (native).
---
 tests/genpackage.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/genpackage.sh b/tests/genpackage.sh
index b97baea5..152a589a 100755
--- a/tests/genpackage.sh
+++ b/tests/genpackage.sh
@@ -39,6 +39,17 @@ $PACKAGE ($EPOCH$VERSION$REVISION) $DISTRI; urgency=critical
  -- me <guess@who>  Mon, 01 Jan 1980 01:02:02 +0000
 END
 
+mkdir -p "$DIR/debian/source"
+if test -z "$REVISION"; then
+	echo "3.0 (native)" > "$DIR/debian/source/format"
+else
+	echo "3.0 (quilt)" > "$DIR/debian/source/format"
+	orig_tarball="${PACKAGE}_${VERSION}.orig.tar.gz"
+	if test ! -f "$orig_tarball"; then
+		tar czvf "$orig_tarball" --files-from /dev/null
+	fi
+fi
+
 dpkg-source -b "$DIR"
 mkdir -p "$DIR"/debian/tmp/DEBIAN
 touch "$DIR"/debian/tmp/x

From df777780f5c4f99568a484e8a9365fee41b32be9 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 28 Mar 2017 15:15:22 +0200
Subject: [PATCH 06/70] [testsuite] genpackage.sh: Silence output

---
 tests/genpackage.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/genpackage.sh b/tests/genpackage.sh
index 152a589a..4207ed9c 100755
--- a/tests/genpackage.sh
+++ b/tests/genpackage.sh
@@ -50,7 +50,7 @@ else
 	fi
 fi
 
-dpkg-source -b "$DIR"
+dpkg-source -b "$DIR" > /dev/null
 mkdir -p "$DIR"/debian/tmp/DEBIAN
 touch "$DIR"/debian/tmp/x
 mkdir "$DIR"/debian/tmp/a
@@ -67,9 +67,9 @@ for pkg in `grep '^Package: ' debian/control | sed -e 's/^Package: //'` ; do
 	else
 		dpkg-gencontrol -p$pkg
 	fi
-	dpkg --build debian/tmp ..
+	dpkg --build debian/tmp .. > /dev/null
 done
-dpkg-genchanges "$@" > "$OUTPUT".pre
+dpkg-genchanges -q "$@" > "$OUTPUT".pre
 # simulate dpkg-genchanges behaviour currently in sid so the testsuite runs for backports, too
 awk 'BEGIN{inheader=0} /^Files:/ || (inheader && /^ /)  {inheader = 1; next} {inheader = 0 ; print}' "$OUTPUT".pre | sed -e 's/ \+$//' >../"$OUTPUT"
 echo "Files:" >> ../"$OUTPUT"

From 81fc5b59cdc9ba37e2b1d659f01e3c0d7b0c096d Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 22 Aug 2017 15:37:49 +0200
Subject: [PATCH 07/70] [testsuite] Use useful names for .changes files

Use the common naming schema $source_$version_$arch.changes for the name
of the .changes files for testing.
---
 tests/genpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/genpackage.sh b/tests/genpackage.sh
index 4207ed9c..d3516526 100755
--- a/tests/genpackage.sh
+++ b/tests/genpackage.sh
@@ -5,7 +5,7 @@ set -e
 #VERSION=0.9-A:Z+a:z
 #REVISION=-0+aA.9zZ
 if [ "x$OUTPUT" == "x" ] ; then
-	OUTPUT=test.changes
+	OUTPUT=${PACKAGE}_${VERSION}${REVISION}_${ARCH:-$(dpkg-architecture -qDEB_HOST_ARCH)}.changes
 fi
 
 DIR="$PACKAGE-$VERSION"

From 3c4e75fddd7cea33a6bfbbd860c8678fcd171e31 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 17:44:16 +0100
Subject: [PATCH 08/70] [testsuite] Add basic shunit2 based tests

Closes: #857302
Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 tests/Makefile.am                 |  10 ++-
 tests/basic.sh                    | 116 ++++++++++++++++++++++++++++++
 tests/shunit2-helper-functions.sh |  67 +++++++++++++++++
 3 files changed, 192 insertions(+), 1 deletion(-)
 create mode 100755 tests/basic.sh
 create mode 100644 tests/shunit2-helper-functions.sh

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 3f4a0db5..115a567b 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -48,5 +48,13 @@ good.key \
 revoked.key \
 revoked.pkey \
 withsubkeys.key \
-withsubkeys-works.key
+withsubkeys-works.key \
+basic.sh \
+shunit2-helper-functions.sh
 MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+check:
+	./basic.sh
+
+clean-local:
+	rm -rf testrepo testpkgs
diff --git a/tests/basic.sh b/tests/basic.sh
new file mode 100755
index 00000000..f2845be4
--- /dev/null
+++ b/tests/basic.sh
@@ -0,0 +1,116 @@
+#!/bin/sh
+set -u
+
+# Copyright (C) 2017, Benjamin Drung <benjamin.drung@profitbricks.com>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+. "${0%/*}/shunit2-helper-functions.sh"
+
+setUp() {
+	create_repo
+}
+
+tearDown() {
+	check_db
+}
+
+test_empty() {
+	$REPREPRO -b $REPO export
+	call $REPREPRO -b $REPO list buster
+	assertEquals "" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_list() {
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=1.0 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_1.0-1_${ARCH}.deb
+	assertEquals "buster|main|$ARCH: hello 1.0-1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_ls() {
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster EPOCH="1:" VERSION=2.5 REVISION=-3 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.5-3_${ARCH}.deb
+	assertEquals "hello | 1:2.5-3 | buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+}
+
+test_copy() {
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster EPOCH="1:" VERSION=2.5 REVISION=-3 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.5-3_${ARCH}.deb
+	assertEquals "hello | 1:2.5-3 | buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	add_distro bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster hello
+	assertEquals "bullseye|main|$ARCH: hello 1:2.5-3" "$($REPREPRO -b $REPO list bullseye)"
+}
+
+test_copy_existing() {
+	add_distro bullseye
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/sl_3.03-1_${ARCH}.deb
+	assertEquals "sl | 3.03-1 | buster | $ARCH" "$($REPREPRO -b $REPO ls sl)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster sl
+	assertEquals "\
+sl | 3.03-1 |   buster | $ARCH
+sl | 3.03-1 | bullseye | $ARCH" "$($REPREPRO -b $REPO ls sl)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster sl
+	assertEquals "\
+sl | 3.03-1 |   buster | $ARCH
+sl | 3.03-1 | bullseye | $ARCH" "$($REPREPRO -b $REPO ls sl)"
+}
+
+test_include_changes() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
+	assertEquals "\
+buster|main|$ARCH: sl 3.03-1
+buster|main|$ARCH: sl-addons 3.03-1
+buster|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_include_old() {
+	# Test including an old package version. Expected output:
+	# Skipping inclusion of 'hello' '2.9-1' in 'buster|main|$ARCH', as it has already '2.9-2'.
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-2_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-1_${ARCH}.deb
+	assertEquals "buster|main|$ARCH: hello 2.9-2" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_limit() {
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-1_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-2_${ARCH}.deb
+	assertEquals "buster|main|$ARCH: hello 2.9-2" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_remove() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
+	assertEquals "\
+buster|main|$ARCH: sl 3.03-1
+buster|main|$ARCH: sl-addons 3.03-1
+buster|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list buster)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO remove buster sl
+	assertEquals "buster|main|$ARCH: sl-addons 3.03-1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_listcodenames() {
+	assertEquals "buster" "$($REPREPRO -b $REPO _listcodenames)"
+	add_distro bullseye
+	assertEquals "\
+buster
+bullseye" "$($REPREPRO -b $REPO _listcodenames)"
+}
+
+. shunit2
diff --git a/tests/shunit2-helper-functions.sh b/tests/shunit2-helper-functions.sh
new file mode 100644
index 00000000..e99dc972
--- /dev/null
+++ b/tests/shunit2-helper-functions.sh
@@ -0,0 +1,67 @@
+# Copyright (C) 2017, Benjamin Drung <benjamin.drung@profitbricks.com>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+REPO="${0%/*}/testrepo"
+PKGS="${0%/*}/testpkgs"
+ARCH=${ARCH:-$(dpkg-architecture -qDEB_HOST_ARCH)}
+REPREPRO=$(realpath -m "${0%/*}/.." --relative-base=.)/reprepro
+VERBOSE_ARGS="${VERBOSE_ARGS-}"
+
+call() {
+	command="$@"
+	echo "I: Calling $@"
+	"$@" || fail "Command '$command' failed with exit code $?."
+}
+
+check_db() {
+	db_verify $REPO/db/packages.db || fail "BerkeleyDB 'packages.db' is broken."
+}
+
+add_distro() {
+	local name="$1"
+	if test -e $REPO/conf/distributions; then
+		echo >> $REPO/conf/distributions
+	fi
+	cat >> $REPO/conf/distributions <<EOF
+Codename: $name
+Architectures: $ARCH source
+Components: main non-free
+Log: testrepo.log
+Tracking: all
+EOF
+	if test -n "${2-}"; then
+		echo "$2" >> $REPO/conf/distributions
+	fi
+}
+
+clear_distro() {
+	rm -f $REPO/conf/distributions
+}
+
+create_repo() {
+	rm -rf $REPO
+	mkdir -p $REPO/conf
+	add_distro buster
+	mkdir -p $PKGS
+	$REPREPRO -b $REPO export
+}
+
+# See https://github.com/wting/shunit2/issues/23
+if test -n "${TEST_CASES-}"; then
+	suite() {
+		for testcase in "${TEST_CASES}" ; do
+			suite_addTest $testcase
+		done
+	}
+fi

From 35142031d963b007d1036e1da479f506acb8b0d7 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 18:26:51 +0100
Subject: [PATCH 09/70] Add ignores for tests to .gitignore

---
 .gitignore | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitignore b/.gitignore
index 2574042a..20d2586a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,8 @@ Makefile
 /debian/reprepro.debhelper.*
 /debian/reprepro.substvars
 /debian/reprepro
+/autom4te.cache/
+/config.h.in~
+/tests/.deps/
+/tests/testpkgs/
+/tests/testrepo/

From ae4a8ff19b6393eb784ded69c82b7471aeb8818a Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 30 May 2014 22:24:22 +0200
Subject: [PATCH 10/70] [format] Fix indentation (spaces to tabs)

---
 main.c | 60 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/main.c b/main.c
index 31aa5cf3..e25f5928 100644
--- a/main.c
+++ b/main.c
@@ -2519,54 +2519,54 @@ ACTION_F(y, n, y, y, reoverride) {
 /*****************retrieving Description data from .deb files***************/
 
 static retvalue repair_descriptions(struct target *target) {
-        struct package_cursor iterator;
-        retvalue result, r;
+	struct package_cursor iterator;
+	retvalue result, r;
 
-        assert(target->packages == NULL);
+	assert(target->packages == NULL);
 	assert(target->packagetype == pt_deb || target->packagetype == pt_udeb);
 
-        if (verbose > 2) {
-                printf(
+	if (verbose > 2) {
+		printf(
 "Redoing checksum information for packages in '%s'...\n",
-                                target->identifier);
-        }
+				target->identifier);
+	}
 
-        r = package_openiterator(target, READWRITE, &iterator);
-        if (!RET_IS_OK(r))
-                return r;
-        result = RET_NOTHING;
-        while (package_next(&iterator)) {
-                char *newcontrolchunk = NULL;
+	r = package_openiterator(target, READWRITE, &iterator);
+	if (!RET_IS_OK(r))
+		return r;
+	result = RET_NOTHING;
+	while (package_next(&iterator)) {
+		char *newcontrolchunk = NULL;
 
 		if (interrupted()) {
 			result = RET_ERROR_INTERRUPTED;
 			break;
 		}
 		/* replace it by itself to normalize the Description field */
-                r = description_addpackage(target, iterator.current.name,
+		r = description_addpackage(target, iterator.current.name,
 				iterator.current.control,
 				&newcontrolchunk);
-                RET_UPDATE(result, r);
-                if (RET_WAS_ERROR(r))
-                        break;
-                if (RET_IS_OK(r)) {
+		RET_UPDATE(result, r);
+		if (RET_WAS_ERROR(r))
+			break;
+		if (RET_IS_OK(r)) {
 			if (verbose >= 0) {
 				printf(
 "Fixing description for '%s'...\n", iterator.current.name);
 			}
 			r = package_newcontrol_by_cursor(&iterator,
-                                newcontrolchunk, strlen(newcontrolchunk));
-                        free(newcontrolchunk);
-                        if (RET_WAS_ERROR(r)) {
-                                result = r;
-                                break;
-                        }
-                        target->wasmodified = true;
-                }
-        }
-        r = package_closeiterator(&iterator);
-        RET_ENDUPDATE(result, r);
-        return result;
+				newcontrolchunk, strlen(newcontrolchunk));
+			free(newcontrolchunk);
+			if (RET_WAS_ERROR(r)) {
+				result = r;
+				break;
+			}
+			target->wasmodified = true;
+		}
+	}
+	r = package_closeiterator(&iterator);
+	RET_ENDUPDATE(result, r);
+	return result;
 }
 
 ACTION_F(y, n, y, y, repairdescriptions) {

From 17e12ba670dbcc05e9f40a10247b21975da9a905 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 12:16:49 +0100
Subject: [PATCH 11/70] Evaluate return value of write command
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Compiling reprepro produces this warning:

```
signature.c: In function ‘signature_getpassphrase’:
signature.c:63:2: warning: ignoring return value of ‘write’, declared with attribute warn_unused_result [-Wunused-result]
  write(fd, p, strlen(p));
  ^~~~~~~~~~~~~~~~~~~~~~~
signature.c:64:2: warning: ignoring return value of ‘write’, declared with attribute warn_unused_result [-Wunused-result]
  write(fd, "\n", 1);
  ^~~~~~~~~~~~~~~~~~
```
---
 signature.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/signature.c b/signature.c
index 991fec24..93ff207d 100644
--- a/signature.c
+++ b/signature.c
@@ -53,6 +53,7 @@ retvalue gpgerror(gpg_error_t err) {
 static gpg_error_t signature_getpassphrase(UNUSED(void *hook), const char *uid_hint, UNUSED(const char *info), int prev_was_bad, int fd) {
 	char *msg;
 	const char *p;
+	int e = 0;
 
 	msg = mprintf("%s needs a passphrase\nPlease enter passphrase%s:",
 			(uid_hint!=NULL)?uid_hint:"key",
@@ -60,8 +61,18 @@ static gpg_error_t signature_getpassphrase(UNUSED(void *hook), const char *uid_h
 	if (msg == NULL)
 		return gpg_err_make(GPG_ERR_SOURCE_USER_1, GPG_ERR_ENOMEM);
 	p = getpass(msg);
-	write(fd, p, strlen(p));
-	write(fd, "\n", 1);
+	if (write(fd, p, strlen(p)) < 0) {
+		e = errno;
+	}
+	if (write(fd, "\n", 1) < 0 && e == 0) {
+		e = errno;
+	}
+	if (e != 0) {
+		fprintf(stderr, "Error %d writing to fd %i: %s\n",
+				e, fd, strerror(e));
+		free(msg);
+		return RET_ERRNO(e);
+	}
 	free(msg);
 	return GPG_ERR_NO_ERROR;
 }

From 34a45293b2bd062ead40dacf0c99c34fb5bbbfbc Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 30 Mar 2017 11:58:29 +0200
Subject: [PATCH 12/70] Fix typo "could not found" -> "could not find"

---
 tests/trackingcorruption.test | 2 +-
 tracking.c                    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/trackingcorruption.test b/tests/trackingcorruption.test
index de62d62e..6110e26c 100644
--- a/tests/trackingcorruption.test
+++ b/tests/trackingcorruption.test
@@ -43,7 +43,7 @@ EOF
 
 testrun - --keepunreferenced remove breakme aa aa-addons 3<<EOF
 stderr
-*=Could not found tracking data for aa_1-1 in breakme to remove old files from it.
+*=Could not find tracking data for aa_1-1 in breakme to remove old files from it.
 stdout
 $(opd 'aa' unset breakme something abacus deb)
 $(opd 'aa-addons' unset breakme something abacus deb)
diff --git a/tracking.c b/tracking.c
index dc33292a..a38ec369 100644
--- a/tracking.c
+++ b/tracking.c
@@ -870,7 +870,7 @@ retvalue trackingdata_insert(struct trackingdata *data, enum filetype filetype,
 	}
 	if (r == RET_NOTHING) {
 		fprintf(stderr,
-"Could not found tracking data for %s_%s in %s to remove old files from it.\n",
+"Could not find tracking data for %s_%s in %s to remove old files from it.\n",
 			old->source, old->sourceversion,
 			data->tracks->codename);
 		return result;
@@ -904,7 +904,7 @@ retvalue trackingdata_remove(struct trackingdata *data, const char* oldsource, c
 	}
 	if (result == RET_NOTHING) {
 		fprintf(stderr,
-"Could not found tracking data for %s_%s in %s to remove old files from it.\n",
+"Could not find tracking data for %s_%s in %s to remove old files from it.\n",
 			oldsource, oldversion, data->tracks->codename);
 		return RET_OK;
 	}

From 00ec33982fa775a26b379f861a46bbc9929b01f8 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 27 Aug 2018 13:25:26 +0200
Subject: [PATCH 13/70] Fix missing quotation mark in component list

The error message in guess_component misses a leading quotation mark,
for example:

Could not find 'main' in components of 'bionic': contrib'
---
 guesscomponent.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guesscomponent.c b/guesscomponent.c
index 9972372b..adb3038a 100644
--- a/guesscomponent.c
+++ b/guesscomponent.c
@@ -39,7 +39,7 @@ retvalue guess_component(const char *codename, const struct atomlist *components
 	if (atom_defined(givencomponent)) {
 		if (!atomlist_in(components, givencomponent)) {
 			(void)fprintf(stderr,
-"Could not find '%s' in components of '%s': ",
+"Could not find '%s' in components of '%s': '",
 					atoms_components[givencomponent],
 					codename);
 			(void)atomlist_fprint(stderr,

From bd8ce63cda5d76201b3105249d9eff4695713747 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 10:16:21 +0100
Subject: [PATCH 14/70] Print version when removing a package

---
 target.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/target.c b/target.c
index 44424cd2..86b2eae3 100644
--- a/target.c
+++ b/target.c
@@ -241,8 +241,8 @@ retvalue package_remove(struct package *old, struct logger *logger, struct track
 		(void)package_getsource(old);
 	}
 	if (verbose > 0)
-		printf("removing '%s' from '%s'...\n",
-				old->name, old->target->identifier);
+		printf("removing '%s=%s' from '%s'...\n",
+				old->name, old->version, old->target->identifier);
 	result = table_deleterecord(old->target->packages, old->name, false);
 	if (RET_IS_OK(result)) {
 		old->target->wasmodified = true;
@@ -301,7 +301,7 @@ retvalue package_remove_by_cursor(struct package_cursor *tc, struct logger *logg
 	assert (target != NULL && target->packages != NULL);
 	assert (target == old->target);
 
-	if (logger != NULL) {
+	if (logger != NULL || verbose > 0) {
 		(void)package_getversion(old);
 	}
 	r = old->target->getfilekeys(old->control, &files);
@@ -312,8 +312,8 @@ retvalue package_remove_by_cursor(struct package_cursor *tc, struct logger *logg
 		(void)package_getsource(old);
 	}
 	if (verbose > 0)
-		printf("removing '%s' from '%s'...\n",
-				old->name, old->target->identifier);
+		printf("removing '%s=%s' from '%s'...\n",
+				old->name, old->version, old->target->identifier);
 	result = cursor_delete(target->packages, tc->cursor, old->name, NULL);
 	if (RET_IS_OK(result)) {
 		old->target->wasmodified = true;

From c3d8dcb7f08382d18f761d9bd157e3cc0842fda5 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 16:45:42 +0100
Subject: [PATCH 15/70] [refactor] Introduce static newcursor() function

There are multiple places where new cursors are generated. Remove
duplicate code by introducing the newcursor() function.
---
 database.c | 63 +++++++++++++++++++-----------------------------------
 1 file changed, 22 insertions(+), 41 deletions(-)

diff --git a/database.c b/database.c
index 7f852e76..a2fb9c10 100644
--- a/database.c
+++ b/database.c
@@ -1299,14 +1299,14 @@ struct cursor {
 	retvalue r;
 };
 
-retvalue table_newglobalcursor(struct table *table, struct cursor **cursor_p) {
+static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **cursor_p) {
 	struct cursor *cursor;
 	int dbret;
 
 	if (table->berkeleydb == NULL) {
 		assert (table->readonly);
 		*cursor_p = NULL;
-		return RET_OK;
+		return RET_NOTHING;
 	}
 
 	cursor = zNEW(struct cursor);
@@ -1314,7 +1314,7 @@ retvalue table_newglobalcursor(struct table *table, struct cursor **cursor_p) {
 		return RET_ERROR_OOM;
 
 	cursor->cursor = NULL;
-	cursor->flags = DB_NEXT;
+	cursor->flags = flags;
 	cursor->r = RET_OK;
 	dbret = table->berkeleydb->cursor(table->berkeleydb, NULL,
 			&cursor->cursor, 0);
@@ -1327,6 +1327,17 @@ retvalue table_newglobalcursor(struct table *table, struct cursor **cursor_p) {
 	return RET_OK;
 }
 
+retvalue table_newglobalcursor(struct table *table, struct cursor **cursor_p) {
+	retvalue r;
+
+	r = newcursor(table, DB_NEXT, cursor_p);
+	if (r == RET_NOTHING) {
+		// table_newglobalcursor returned RET_OK when table->berkeleydb == NULL. Is that return value wanted?
+		r = RET_OK;
+	}
+	return r;
+}
+
 static inline retvalue parse_pair(struct table *table, DBT Key, DBT Data, /*@null@*//*@out@*/const char **key_p, /*@out@*/const char **value_p, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) {
 	/*@dependant@*/ const char *separator;
 
@@ -1369,26 +1380,11 @@ retvalue table_newduplicatecursor(struct table *table, const char *key, struct c
 	DBT Key, Data;
 	retvalue r;
 
-	if (table->berkeleydb == NULL) {
-		assert (table->readonly);
-		*cursor_p = NULL;
-		return RET_NOTHING;
-	}
-
-	cursor = zNEW(struct cursor);
-	if (FAILEDTOALLOC(cursor))
-		return RET_ERROR_OOM;
-
-	cursor->cursor = NULL;
-	cursor->flags = DB_NEXT_DUP;
-	cursor->r = RET_OK;
-	dbret = table->berkeleydb->cursor(table->berkeleydb, NULL,
-			&cursor->cursor, 0);
-	if (dbret != 0) {
-		table_printerror(table, dbret, "cursor");
-		free(cursor);
-		return RET_DBERR(dbret);
+	r = newcursor(table, DB_NEXT_DUP, cursor_p);
+	if(!RET_IS_OK(r)) {
+		return r;
 	}
+	cursor = *cursor_p;
 	SETDBT(Key, key);
 	CLEARDBT(Data);
 	dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_SET);
@@ -1422,27 +1418,12 @@ retvalue table_newpairedcursor(struct table *table, const char *key, const char
 	retvalue r;
 	size_t valuelen = strlen(value);
 
-	if (table->berkeleydb == NULL) {
-		assert (table->readonly);
-		*cursor_p = NULL;
-		return RET_NOTHING;
-	}
-
-	cursor = zNEW(struct cursor);
-	if (FAILEDTOALLOC(cursor))
-		return RET_ERROR_OOM;
-
-	cursor->cursor = NULL;
 	/* cursor_next is not allowed with this type: */
-	cursor->flags = DB_GET_BOTH;
-	cursor->r = RET_OK;
-	dbret = table->berkeleydb->cursor(table->berkeleydb, NULL,
-			&cursor->cursor, 0);
-	if (dbret != 0) {
-		table_printerror(table, dbret, "cursor");
-		free(cursor);
-		return RET_DBERR(dbret);
+	r = newcursor(table, DB_GET_BOTH, cursor_p);
+	if(!RET_IS_OK(r)) {
+		return r;
 	}
+	cursor = *cursor_p;
 	SETDBT(Key, key);
 	SETDBTl(Data, value, valuelen + 1);
 	dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_GET_BOTH);

From 8e330e4fa8c05f504087094f5e43a55607d5850a Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 1 Feb 2017 11:25:14 +0100
Subject: [PATCH 16/70] table_getrecord: Add newline to error message

---
 database.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/database.c b/database.c
index a2fb9c10..e5ada0d4 100644
--- a/database.c
+++ b/database.c
@@ -1019,11 +1019,11 @@ retvalue table_getrecord(struct table *table, const char *key, char **data_p, si
 	    ((const char*)Data.data)[Data.size-1] != '\0') {
 		if (table->subname != NULL)
 			fprintf(stderr,
-"Database %s(%s) returned corrupted (not null-terminated) data!",
+"Database %s(%s) returned corrupted (not null-terminated) data!\n",
 					table->name, table->subname);
 		else
 			fprintf(stderr,
-"Database %s returned corrupted (not null-terminated) data!",
+"Database %s returned corrupted (not null-terminated) data!\n",
 					table->name);
 		free(Data.data);
 		return RET_ERROR;
@@ -1107,11 +1107,11 @@ retvalue table_gettemprecord(struct table *table, const char *key, const char **
 	    ((const char*)Data.data)[Data.size-1] != '\0') {
 		if (table->subname != NULL)
 			fprintf(stderr,
-"Database %s(%s) returned corrupted (not null-terminated) data!",
+"Database %s(%s) returned corrupted (not null-terminated) data!\n",
 					table->name, table->subname);
 		else
 			fprintf(stderr,
-"Database %s returned corrupted (not null-terminated) data!",
+"Database %s returned corrupted (not null-terminated) data!\n",
 					table->name);
 		return RET_ERROR;
 	}
@@ -1358,11 +1358,11 @@ static inline retvalue parse_pair(struct table *table, DBT Key, DBT Data, /*@nul
 	if (separator == NULL) {
 		if (table->subname != NULL)
 			fprintf(stderr,
-"Database %s(%s) returned corrupted data!",
+"Database %s(%s) returned corrupted data!\n",
 					table->name, table->subname);
 		else
 			fprintf(stderr,
-"Database %s returned corrupted data!",
+"Database %s returned corrupted data!\n",
 					table->name);
 		return RET_ERROR;
 	}

From c73a4be89f63d6d44f744729f4bb2b6d3419d669 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 15:37:20 +0100
Subject: [PATCH 17/70] [refactor] Merge cursor_nexttemp() into
 cursor_nexttempdata()

cursor_nexttempdata has an additional len_p output parameter (compared
to cursor_nexttemp). Make the len_p output parameter optional and
replace cursor_nexttemp by cursor_nexttempdata.

Thus cursor_nexttemp(...) becomes cursor_nexttempdata(..., NULL).
---
 database.c  | 53 ++++++++---------------------------------------------
 database.h  |  1 -
 files.c     |  6 +++---
 main.c      |  2 +-
 reference.c |  4 ++--
 5 files changed, 14 insertions(+), 52 deletions(-)

diff --git a/database.c b/database.c
index e5ada0d4..9514651d 100644
--- a/database.c
+++ b/database.c
@@ -1477,44 +1477,6 @@ retvalue cursor_close(struct table *table, struct cursor *cursor) {
 	return r;
 }
 
-bool cursor_nexttemp(struct table *table, struct cursor *cursor, const char **key, const char **data) {
-	DBT Key, Data;
-	int dbret;
-
-	if (cursor == NULL)
-		return false;
-
-	CLEARDBT(Key);
-	CLEARDBT(Data);
-
-	dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_NEXT);
-	if (dbret == DB_NOTFOUND)
-		return false;
-
-	if (dbret != 0) {
-		table_printerror(table, dbret, "c_get(DB_NEXT)");
-		cursor->r = RET_DBERR(dbret);
-		return false;
-	}
-	if (Key.size <= 0 || Data.size <= 0 ||
-	    ((const char*)Key.data)[Key.size-1] != '\0' ||
-	    ((const char*)Data.data)[Data.size-1] != '\0') {
-		if (table->subname != NULL)
-			fprintf(stderr,
-"Database %s(%s) returned corrupted (not null-terminated) data!",
-					table->name, table->subname);
-		else
-			fprintf(stderr,
-"Database %s returned corrupted (not null-terminated) data!",
-					table->name);
-		cursor->r = RET_ERROR;
-		return false;
-	}
-	*key = Key.data;
-	*data = Data.data;
-	return true;
-}
-
 bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char **key, const char **data, size_t *len_p) {
 	DBT Key, Data;
 	int dbret;
@@ -1551,7 +1513,8 @@ bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char
 	if (key != NULL)
 		*key = Key.data;
 	*data = Data.data;
-	*len_p = Data.size - 1;
+	if (len_p != NULL)
+		*len_p = Data.size - 1;
 	return true;
 }
 
@@ -2077,8 +2040,8 @@ static inline retvalue translate(struct table *oldmd5sums, struct table *newchec
 	r = table_newglobalcursor(oldmd5sums, &cursor);
 	if (RET_WAS_ERROR(r))
 		return r;
-	while (cursor_nexttemp(oldmd5sums, cursor,
-				&filekey, &md5sum)) {
+	while (cursor_nexttempdata(oldmd5sums, cursor,
+				&filekey, &md5sum, NULL)) {
 		struct checksums *n = NULL;
 		const char *combined;
 		size_t combinedlen;
@@ -2145,15 +2108,15 @@ static inline retvalue translate(struct table *oldmd5sums, struct table *newchec
 		cursor_close(oldmd5sums, cursor);
 		return r;
 	}
-	while (cursor_nexttemp(oldmd5sums, cursor,
-				&filekey, &md5sum)) {
+	while (cursor_nexttempdata(oldmd5sums, cursor,
+				&filekey, &md5sum, NULL)) {
 		bool more;
 		int cmp;
 		const char *newfilekey, *dummy;
 
 		do {
-			more = cursor_nexttemp(newchecksums, newcursor,
-				&newfilekey, &dummy);
+			more = cursor_nexttempdata(newchecksums, newcursor,
+				&newfilekey, &dummy, NULL);
 			/* should have been added in the last step */
 			assert (more);
 			cmp = strcmp(filekey, newfilekey);
diff --git a/database.h b/database.h
index e4791949..b9a913e2 100644
--- a/database.h
+++ b/database.h
@@ -50,7 +50,6 @@ retvalue table_removerecord(struct table *, const char *key, const char *data);
 retvalue table_newglobalcursor(struct table *, /*@out@*/struct cursor **);
 retvalue table_newduplicatecursor(struct table *, const char *, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue table_newpairedcursor(struct table *, const char *, const char *, /*@out@*/struct cursor **, /*@out@*//*@null@*/const char **, /*@out@*//*@null@*/size_t *);
-bool cursor_nexttemp(struct table *, struct cursor *, /*@out@*/const char **, /*@out@*/const char **);
 bool cursor_nexttempdata(struct table *, struct cursor *, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 bool cursor_nextpair(struct table *, struct cursor *, /*@null@*//*@out@*/const char **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue cursor_replace(struct table *, struct cursor *, const char *, size_t);
diff --git a/files.c b/files.c
index 0d2174f8..f378b70a 100644
--- a/files.c
+++ b/files.c
@@ -280,7 +280,7 @@ retvalue files_printmd5sums(void) {
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
-	while (cursor_nexttemp(rdb_checksums, cursor, &filekey, &checksum)) {
+	while (cursor_nexttempdata(rdb_checksums, cursor, &filekey, &checksum, NULL)) {
 		result = RET_OK;
 		(void)fputs(filekey, stdout);
 		(void)putchar(' ');
@@ -307,7 +307,7 @@ retvalue files_printchecksums(void) {
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
-	while (cursor_nexttemp(rdb_checksums, cursor, &filekey, &checksum)) {
+	while (cursor_nexttempdata(rdb_checksums, cursor, &filekey, &checksum, NULL)) {
 		result = RET_OK;
 		(void)fputs(filekey, stdout);
 		(void)putchar(' ');
@@ -333,7 +333,7 @@ retvalue files_foreach(per_file_action action, void *privdata) {
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
-	while (cursor_nexttemp(rdb_checksums, cursor, &filekey, &checksum)) {
+	while (cursor_nexttempdata(rdb_checksums, cursor, &filekey, &checksum, NULL)) {
 		if (interrupted()) {
 			RET_UPDATE(result, RET_ERROR_INTERRUPTED);
 			break;
diff --git a/main.c b/main.c
index e25f5928..2a3e61da 100644
--- a/main.c
+++ b/main.c
@@ -1576,7 +1576,7 @@ ACTION_B(n, n, n, dumpcontents) {
 		return r;
 	}
 	result = RET_NOTHING;
-	while (cursor_nexttemp(packages, cursor, &package, &chunk)) {
+	while (cursor_nexttempdata(packages, cursor, &package, &chunk, NULL)) {
 		printf("'%s' -> '%s'\n", package, chunk);
 		result = RET_OK;
 	}
diff --git a/reference.c b/reference.c
index 65e1ca1f..563cfe76 100644
--- a/reference.c
+++ b/reference.c
@@ -193,8 +193,8 @@ retvalue references_dump(void) {
 		return r;
 
 	result = RET_OK;
-	while (cursor_nexttemp(rdb_references, cursor,
-	                               &found_to, &found_by)) {
+	while (cursor_nexttempdata(rdb_references, cursor,
+	                               &found_to, &found_by, NULL)) {
 		if (fputs(found_by, stdout) == EOF ||
 		    putchar(' ') == EOF ||
 		    puts(found_to) == EOF) {

From 689daf408ccf6f6d3b3c94682c86a83c7c782436 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 16:01:33 +0100
Subject: [PATCH 18/70] [refactor] Introduce parse_data()

The cursor_nextpair() function has a parse_pair() function for
evaluating the returned database output. Introduce a similar
parse_data() function for the cursor_nexttempdata() function.
---
 database.c | 43 ++++++++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/database.c b/database.c
index 9514651d..fd01a85b 100644
--- a/database.c
+++ b/database.c
@@ -1338,6 +1338,28 @@ retvalue table_newglobalcursor(struct table *table, struct cursor **cursor_p) {
 	return r;
 }
 
+static inline retvalue parse_data(struct table *table, DBT Key, DBT Data, /*@null@*//*@out@*/const char **key_p, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) {
+	if (Key.size <= 0 || Data.size <= 0 ||
+	    ((const char*)Key.data)[Key.size-1] != '\0' ||
+	    ((const char*)Data.data)[Data.size-1] != '\0') {
+		if (table->subname != NULL)
+			fprintf(stderr,
+"Database %s(%s) returned corrupted (not null-terminated) data!",
+					table->name, table->subname);
+		else
+			fprintf(stderr,
+"Database %s returned corrupted (not null-terminated) data!",
+					table->name);
+		return RET_ERROR;
+	}
+	if (key_p != NULL)
+		*key_p = Key.data;
+	*data_p = Data.data;
+	if (datalen_p != NULL)
+		*datalen_p = Data.size - 1;
+	return RET_OK;
+}
+
 static inline retvalue parse_pair(struct table *table, DBT Key, DBT Data, /*@null@*//*@out@*/const char **key_p, /*@out@*/const char **value_p, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) {
 	/*@dependant@*/ const char *separator;
 
@@ -1480,6 +1502,7 @@ retvalue cursor_close(struct table *table, struct cursor *cursor) {
 bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char **key, const char **data, size_t *len_p) {
 	DBT Key, Data;
 	int dbret;
+	retvalue r;
 
 	if (cursor == NULL)
 		return false;
@@ -1496,25 +1519,11 @@ bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char
 		cursor->r = RET_DBERR(dbret);
 		return false;
 	}
-	if (Key.size <= 0 || Data.size <= 0 ||
-	    ((const char*)Key.data)[Key.size-1] != '\0' ||
-	    ((const char*)Data.data)[Data.size-1] != '\0') {
-		if (table->subname != NULL)
-			fprintf(stderr,
-"Database %s(%s) returned corrupted (not null-terminated) data!",
-					table->name, table->subname);
-		else
-			fprintf(stderr,
-"Database %s returned corrupted (not null-terminated) data!",
-					table->name);
-		cursor->r = RET_ERROR;
+	r = parse_data(table, Key, Data, key, data, len_p);
+	if (RET_WAS_ERROR(r)) {
+		cursor->r = r;
 		return false;
 	}
-	if (key != NULL)
-		*key = Key.data;
-	*data = Data.data;
-	if (len_p != NULL)
-		*len_p = Data.size - 1;
 	return true;
 }
 

From 1af26e480341e2ed8505261bd8cdeb17607e6da8 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 1 Feb 2017 10:28:17 +0100
Subject: [PATCH 19/70] table_printerror: Improve database error message

---
 database.c | 41 ++++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/database.c b/database.c
index fd01a85b..427d961b 100644
--- a/database.c
+++ b/database.c
@@ -953,15 +953,38 @@ struct table {
 };
 
 static void table_printerror(struct table *table, int dbret, const char *action) {
-	if (table->subname != NULL)
-		table->berkeleydb->err(table->berkeleydb, dbret,
-				"%sWithin %s subtable %s at %s",
-				databaseerror, table->name, table->subname,
-				action);
-	else
-		table->berkeleydb->err(table->berkeleydb, dbret,
-				"%sWithin %s at %s",
-				databaseerror, table->name, action);
+	char *error_msg;
+
+	switch (dbret) {
+	case RET_ERROR_OOM:
+		error_msg = "RET_ERROR_OOM: Out of memory.";
+		break;
+	default:
+		error_msg = NULL;
+		break;
+	}
+
+	if (error_msg == NULL) {
+		if (table->subname != NULL)
+			table->berkeleydb->err(table->berkeleydb, dbret,
+					"%sWithin %s subtable %s at %s",
+					databaseerror, table->name, table->subname,
+					action);
+		else
+			table->berkeleydb->err(table->berkeleydb, dbret,
+					"%sWithin %s at %s",
+					databaseerror, table->name, action);
+	} else {
+		if (table->subname != NULL)
+			table->berkeleydb->errx(table->berkeleydb,
+					"%sWithin %s subtable %s at %s: %s",
+					databaseerror, table->name, table->subname,
+					action, error_msg);
+		else
+			table->berkeleydb->errx(table->berkeleydb,
+					"%sWithin %s at %s: %s",
+					databaseerror, table->name, action, error_msg);
+	}
 }
 
 retvalue table_close(struct table *table) {

From 6608f0870f4b62252ae190aa79a421b96d2c8afe Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 14:48:18 +0100
Subject: [PATCH 20/70] [refactor] database: Move cursor struct upwards

Move cursor struct upwards to have the struct definition in one block.
---
 database.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/database.c b/database.c
index 427d961b..c72c05a3 100644
--- a/database.c
+++ b/database.c
@@ -945,6 +945,12 @@ static const char databaseerror[] = "Internal error of the underlying BerkeleyDB
  There is nothing that cannot be solved by another layer of indirection, except
  too many levels of indirection. (Source forgotten) */
 
+struct cursor {
+	DBC *cursor;
+	uint32_t flags;
+	retvalue r;
+};
+
 struct table {
 	char *name, *subname;
 	DB *berkeleydb;
@@ -1316,12 +1322,6 @@ retvalue table_replacerecord(struct table *table, const char *key, const char *d
 	return table_adduniqrecord(table, key, data);
 }
 
-struct cursor {
-	DBC *cursor;
-	uint32_t flags;
-	retvalue r;
-};
-
 static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **cursor_p) {
 	struct cursor *cursor;
 	int dbret;

From 0f5d3f9c4788523ef01fee38c14aa1a8beec126e Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 17:56:13 +0100
Subject: [PATCH 21/70] [refactor] rename table_newduplicatecursor

Rename table_newduplicatecursor to table_newduplicatepairedcursor to
make use this name for a data cursor.
---
 database.c | 2 +-
 database.h | 2 +-
 tracking.c | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/database.c b/database.c
index c72c05a3..dab26879 100644
--- a/database.c
+++ b/database.c
@@ -1419,7 +1419,7 @@ static inline retvalue parse_pair(struct table *table, DBT Key, DBT Data, /*@nul
 	return RET_OK;
 }
 
-retvalue table_newduplicatecursor(struct table *table, const char *key, struct cursor **cursor_p, const char **value_p, const char **data_p, size_t *datalen_p) {
+retvalue table_newduplicatepairedcursor(struct table *table, const char *key, struct cursor **cursor_p, const char **value_p, const char **data_p, size_t *datalen_p) {
 	struct cursor *cursor;
 	int dbret;
 	DBT Key, Data;
diff --git a/database.h b/database.h
index b9a913e2..44293c4b 100644
--- a/database.h
+++ b/database.h
@@ -48,7 +48,7 @@ retvalue table_checkrecord(struct table *, const char *key, const char *data);
 retvalue table_removerecord(struct table *, const char *key, const char *data);
 
 retvalue table_newglobalcursor(struct table *, /*@out@*/struct cursor **);
-retvalue table_newduplicatecursor(struct table *, const char *, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
+retvalue table_newduplicatepairedcursor(struct table *, const char *, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue table_newpairedcursor(struct table *, const char *, const char *, /*@out@*/struct cursor **, /*@out@*//*@null@*/const char **, /*@out@*//*@null@*/size_t *);
 bool cursor_nexttempdata(struct table *, struct cursor *, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 bool cursor_nextpair(struct table *, struct cursor *, /*@null@*//*@out@*/const char **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
diff --git a/tracking.c b/tracking.c
index a38ec369..55ab89f8 100644
--- a/tracking.c
+++ b/tracking.c
@@ -1139,7 +1139,7 @@ static retvalue tracking_foreachversion(trackingdb t, struct distribution *distr
 	const char *value, *data;
 	size_t datalen;
 
-	r = table_newduplicatecursor(t->table, sourcename, &cursor,
+	r = table_newduplicatepairedcursor(t->table, sourcename, &cursor,
 			&value, &data, &datalen);
 	if (!RET_IS_OK(r))
 		return r;

From 572e2e002bbbba46a3b8c050261b6a4fdbc26803 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 19:27:07 +0100
Subject: [PATCH 22/70] [refactor] Introduce cursor_next()

The functions cursor_nexttempdata() and cursor_nextpair() share a
similar logic. Thus combine the duplicate code in cursor_next().

cursor_nexttempdata() set always DB_NEXT as cursor flag instead of using
the cursor->flags value. All users of cursor_nexttempdata() call
table_newglobalcursor() [1] beforehand which sets cursor->flags = DB_NEXT.

[1] Capsulated dependencies: cursor_nexttempdata -> package_next ->
package_openiterator -> table_newglobalcursor
---
 database.c | 53 +++++++++++++++++++++++++----------------------------
 1 file changed, 25 insertions(+), 28 deletions(-)

diff --git a/database.c b/database.c
index dab26879..ab67b62f 100644
--- a/database.c
+++ b/database.c
@@ -1522,26 +1522,41 @@ retvalue cursor_close(struct table *table, struct cursor *cursor) {
 	return r;
 }
 
-bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char **key, const char **data, size_t *len_p) {
-	DBT Key, Data;
+static bool cursor_next(struct table *table, struct cursor *cursor, DBT *Key, DBT *Data) {
 	int dbret;
-	retvalue r;
 
 	if (cursor == NULL)
 		return false;
 
-	CLEARDBT(Key);
-	CLEARDBT(Data);
+	CLEARDBT(*Key);
+	CLEARDBT(*Data);
 
-	dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_NEXT);
+	dbret = cursor->cursor->c_get(cursor->cursor, Key, Data,
+			cursor->flags);
 	if (dbret == DB_NOTFOUND)
 		return false;
 
 	if (dbret != 0) {
-		table_printerror(table, dbret, "c_get(DB_NEXT)");
+		table_printerror(table, dbret,
+				(cursor->flags==DB_NEXT)
+					? "c_get(DB_NEXT)"
+					: (cursor->flags==DB_NEXT_DUP)
+						? "c_get(DB_NEXT_DUP)"
+						: "c_get(DB_???NEXT)");
 		cursor->r = RET_DBERR(dbret);
 		return false;
 	}
+	return true;
+}
+
+bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char **key, const char **data, size_t *len_p) {
+	DBT Key, Data;
+	bool success;
+	retvalue r;
+
+	success = cursor_next(table, cursor, &Key, &Data);
+	if (!success)
+		return false;
 	r = parse_data(table, Key, Data, key, data, len_p);
 	if (RET_WAS_ERROR(r)) {
 		cursor->r = r;
@@ -1552,30 +1567,12 @@ bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char
 
 bool cursor_nextpair(struct table *table, struct cursor *cursor, /*@null@*/const char **key_p, const char **value_p, const char **data_p, size_t *datalen_p) {
 	DBT Key, Data;
-	int dbret;
+	bool success;
 	retvalue r;
 
-	if (cursor == NULL)
-		return false;
-
-	CLEARDBT(Key);
-	CLEARDBT(Data);
-
-	dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data,
-			cursor->flags);
-	if (dbret == DB_NOTFOUND)
-		return false;
-
-	if (dbret != 0) {
-		table_printerror(table, dbret,
-				(cursor->flags==DB_NEXT)
-					? "c_get(DB_NEXT)"
-					: (cursor->flags==DB_NEXT_DUP)
-						? "c_get(DB_NEXT_DUP)"
-						: "c_get(DB_???NEXT)");
-		cursor->r = RET_DBERR(dbret);
+	success = cursor_next(table, cursor, &Key, &Data);
+	if (!success)
 		return false;
-	}
 	r = parse_pair(table, Key, Data, key_p, value_p, data_p, datalen_p);
 	if (RET_WAS_ERROR(r)) {
 		cursor->r = r;

From f503e07d61a1c11be60ee56116a5767eb61ad64c Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 16:27:12 +0100
Subject: [PATCH 23/70] [refactor] rename todo to remaining

The word 'todo' is used for marking todo items for the programmer. Thus
use 'remaining' instead of 'todo' as variable name.
---
 main.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/main.c b/main.c
index 2a3e61da..da9ea527 100644
--- a/main.c
+++ b/main.c
@@ -650,7 +650,7 @@ ACTION_R(n, n, n, y, addreferences) {
 	return ret;
 }
 
-static retvalue remove_from_target(struct distribution *distribution, struct trackingdata *trackingdata, struct target *target, int count, const char * const *names, int *todo, bool *gotremoved) {
+static retvalue remove_from_target(struct distribution *distribution, struct trackingdata *trackingdata, struct target *target, int count, const char * const *names, int *remaining, bool *gotremoved) {
 	retvalue result, r;
 	int i;
 
@@ -661,7 +661,7 @@ static retvalue remove_from_target(struct distribution *distribution, struct tra
 		RET_UPDATE(distribution->status, r);
 		if (RET_IS_OK(r)) {
 			if (!gotremoved[i])
-				(*todo)--;
+				(*remaining)--;
 			gotremoved[i] = true;
 		}
 		RET_UPDATE(result, r);
@@ -674,7 +674,7 @@ ACTION_D(y, n, y, remove) {
 	struct distribution *distribution;
 	struct target *t;
 	bool *gotremoved;
-	int todo;
+	int remaining;
 
 	trackingdb tracks;
 	struct trackingdata trackingdata;
@@ -707,7 +707,7 @@ ACTION_D(y, n, y, remove) {
 		}
 	}
 
-	todo = argc-2;
+	remaining = argc-2;
 	gotremoved = nzNEW(argc - 2, bool);
 	result = RET_NOTHING;
 	if (FAILEDTOALLOC(gotremoved))
@@ -724,7 +724,7 @@ ACTION_D(y, n, y, remove) {
 				 	? &trackingdata
 					: NULL,
 				 t, argc-2, argv+2,
-				 &todo, gotremoved);
+				 &remaining, gotremoved);
 		 RET_UPDATE(result, r);
 		 r = target_closepackagesdb(t);
 		 RET_UPDATE(distribution->status, r);
@@ -741,7 +741,7 @@ ACTION_D(y, n, y, remove) {
 		r = tracking_done(tracks);
 		RET_ENDUPDATE(result, r);
 	}
-	if (verbose >= 0 && !RET_WAS_ERROR(result) && todo > 0) {
+	if (verbose >= 0 && !RET_WAS_ERROR(result) && remaining > 0) {
 		int i = argc - 2;
 
 		(void)fputs("Not removed as not found: ", stderr);
@@ -750,8 +750,8 @@ ACTION_D(y, n, y, remove) {
 			assert(gotremoved != NULL);
 			if (!gotremoved[i]) {
 				(void)fputs(argv[2 + i], stderr);
-				todo--;
-				if (todo > 0)
+				remaining--;
+				if (remaining > 0)
 					(void)fputs(", ", stderr);
 			}
 		}

From 9c232a2a9efc950a7f756305562831d7751bc5ce Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 17:21:45 +0100
Subject: [PATCH 24/70] [refactor] target_removepackage: Support specifying
 package version

If no package version is specfied, use the latest version.
---
 main.c        |  2 +-
 target.c      | 15 ++++++++++-----
 target.h      |  2 +-
 upgradelist.c |  4 ++--
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/main.c b/main.c
index da9ea527..ddff1093 100644
--- a/main.c
+++ b/main.c
@@ -657,7 +657,7 @@ static retvalue remove_from_target(struct distribution *distribution, struct tra
 	result = RET_NOTHING;
 	for (i = 0 ; i < count ; i++){
 		r = target_removepackage(target, distribution->logger,
-				names[i], trackingdata);
+				names[i], NULL, trackingdata);
 		RET_UPDATE(distribution->status, r);
 		if (RET_IS_OK(r)) {
 			if (!gotremoved[i])
diff --git a/target.c b/target.c
index 86b2eae3..7937e131 100644
--- a/target.c
+++ b/target.c
@@ -270,20 +270,25 @@ retvalue package_remove(struct package *old, struct logger *logger, struct track
 }
 
 /* Remove a package from the given target. */
-retvalue target_removepackage(struct target *target, struct logger *logger, const char *name, struct trackingdata *trackingdata) {
+retvalue target_removepackage(struct target *target, struct logger *logger, const char *name, const char *version, struct trackingdata *trackingdata) {
 	struct package old;
 	retvalue r;
 
 	assert(target != NULL && target->packages != NULL && name != NULL);
 
-	r = package_get(target, name, NULL, &old);
+	r = package_get(target, name, version, &old);
 	if (RET_WAS_ERROR(r)) {
 		return r;
 	}
 	else if (r == RET_NOTHING) {
-		if (verbose >= 10)
-			fprintf(stderr, "Could not find '%s' in '%s'...\n",
-					name, target->identifier);
+		if (verbose >= 10) {
+			if (version == NULL)
+				fprintf(stderr, "Could not find '%s' in '%s'...\n",
+						name, target->identifier);
+			else
+				fprintf(stderr, "Could not find '%s=%s' in '%s'...\n",
+						name, version, target->identifier);
+		}
 		return RET_NOTHING;
 	}
 	r = package_remove(&old, logger, trackingdata);
diff --git a/target.h b/target.h
index 1ccb03a2..05792b2d 100644
--- a/target.h
+++ b/target.h
@@ -93,7 +93,7 @@ struct logger;
 struct description;
 retvalue target_addpackage(struct target *, /*@null@*/struct logger *, const char *name, const char *version, const char *control, const struct strlist *filekeys, bool downgrade, /*@null@*/struct trackingdata *, architecture_t, /*@null@*/const char *causingrule, /*@null@*/const char *suitefrom);
 retvalue target_checkaddpackage(struct target *, const char *name, const char *version, bool tracking, bool permitnewerold);
-retvalue target_removepackage(struct target *, /*@null@*/struct logger *, const char *name, struct trackingdata *);
+retvalue target_removepackage(struct target *, /*@null@*/struct logger *, const char *name, const char *version, struct trackingdata *);
 /* like target_removepackage, but do not read control data yourself but use available */
 retvalue target_rereference(struct target *);
 retvalue target_reoverride(struct target *, struct distribution *);
diff --git a/upgradelist.c b/upgradelist.c
index f3f0d7a3..41995c80 100644
--- a/upgradelist.c
+++ b/upgradelist.c
@@ -609,7 +609,7 @@ retvalue upgradelist_predelete(struct upgradelist *upgrade, struct logger *logge
 				r = RET_ERROR_INTERRUPTED;
 			else
 				r = target_removepackage(upgrade->target,
-						logger, pkg->name, NULL);
+						logger, pkg->name, NULL, NULL);
 			RET_UPDATE(result, r);
 			if (RET_WAS_ERROR(r))
 				break;
@@ -713,7 +713,7 @@ retvalue upgradelist_install(struct upgradelist *upgrade, struct logger *logger,
 				r = RET_ERROR_INTERRUPTED;
 			else
 				r = target_removepackage(upgrade->target,
-						logger, pkg->name, NULL);
+						logger, pkg->name, NULL, NULL);
 			RET_UPDATE(result, r);
 			if (RET_WAS_ERROR(r))
 				break;

From 442dc90f2932951626ce15e6efe56951c33dcd97 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 15:40:53 +0100
Subject: [PATCH 25/70] [refactor] Introduce splitnameandversion()

The multi version support will require splitting the name and version in
multiple places. Thus moved the code in a splitnameandversion()
function.
---
 main.c | 96 ++++++++++++++++++++++++++++++++++------------------------
 1 file changed, 57 insertions(+), 39 deletions(-)

diff --git a/main.c b/main.c
index ddff1093..c30da24e 100644
--- a/main.c
+++ b/main.c
@@ -245,6 +245,57 @@ O(fast), O(x_morguedir), O(x_outdir), O(x_basedir), O(x_distdir), O(x_dbdir), O(
 			act(const struct atomlist *, packagetypes),  \
 			u(int, argc), u(const char *, argv[]))
 
+static retvalue splitnameandversion(const char *nameandversion, const char **name_p, const char **version_p) {
+	char *version;
+	retvalue r;
+
+	version = index(nameandversion, '=');
+	if (version != NULL) {
+		if (index(version+1, '=') != NULL) {
+			fprintf(stderr,
+"Cannot parse '%s': more than one '='\n",
+					nameandversion);
+			*name_p = NULL;
+			*version_p = NULL;
+			r = RET_ERROR;
+		} else if (version[1] == '\0') {
+			fprintf(stderr,
+"Cannot parse '%s': no version after '='\n",
+					nameandversion);
+			*name_p = NULL;
+			*version_p = NULL;
+			r = RET_ERROR;
+		} else if (version == nameandversion) {
+			fprintf(stderr,
+"Cannot parse '%s': no source name found before the '='\n",
+					nameandversion);
+			*name_p = NULL;
+			*version_p = NULL;
+			r = RET_ERROR;
+		} else {
+			*name_p = strndup(nameandversion, version - nameandversion);
+			if (FAILEDTOALLOC(*name_p))
+				r = RET_ERROR_OOM;
+			else
+				r = RET_OK;
+			*version_p = version + 1;
+		}
+	} else {
+		r = RET_OK;
+		*name_p = nameandversion;
+		*version_p = NULL;
+	}
+	return r;
+}
+
+static inline void splitnameandversion_done(const char **name_p, const char **version_p) {
+	// In case version_p points to a non-NULL value, name_p needs to be freed after usage.
+	if (*version_p != NULL) {
+		free((char*)*name_p);
+		*name_p = NULL;
+	}
+}
+
 ACTION_N(n, n, y, printargs) {
 	int i;
 
@@ -899,44 +950,12 @@ ACTION_D(n, n, y, removesrcs) {
 	}
 	for (i = 0 ; i < argc-2 ; i++) {
 		data[i].found = false;
-		data[i].sourcename = argv[2 + i];
-		data[i].sourceversion = index(data[i].sourcename, '=');
-		if (data[i].sourceversion != NULL) {
-			if (index(data[i].sourceversion+1, '=') != NULL) {
-				fprintf(stderr,
-"Cannot parse '%s': more than one '='\n",
-						data[i].sourcename);
-				data[i].sourcename = NULL;
-				r = RET_ERROR;
-			} else if (data[i].sourceversion[1] == '\0') {
-				fprintf(stderr,
-"Cannot parse '%s': no version after '='\n",
-						data[i].sourcename);
-				data[i].sourcename = NULL;
-				r = RET_ERROR;
-			} else if (data[i].sourceversion == data[i].sourcename) {
-				fprintf(stderr,
-"Cannot parse '%s': no source name found before the '='\n",
-						data[i].sourcename);
-				data[i].sourcename = NULL;
-				r = RET_ERROR;
-			} else {
-				data[i].sourcename = strndup(data[i].sourcename,
-						data[i].sourceversion
-						- data[i].sourcename);
-				if (FAILEDTOALLOC(data[i].sourcename))
-					r = RET_ERROR_OOM;
-				else
-					r = RET_OK;
-			}
-			if (RET_WAS_ERROR(r)) {
-				for (i-- ; i >= 0 ; i--) {
-					if (data[i].sourceversion != NULL)
-						free((char*)data[i].sourcename);
-				}
-				return r;
+		r = splitnameandversion(argv[2 + i], &data[i].sourcename, &data[i].sourceversion);
+		if (RET_WAS_ERROR(r)) {
+			for (i--; i >= 0; i--) {
+				splitnameandversion_done(&data[i].sourcename, &data[i].sourceversion);
 			}
-			data[i].sourceversion++;
+			return r;
 		}
 	}
 	data[i].sourcename = NULL;
@@ -954,8 +973,7 @@ ACTION_D(n, n, y, removesrcs) {
 "No package from source '%s' (any version) found.\n",
 						data[i].sourcename);
 		}
-		if (data[i].sourceversion != NULL)
-			free((char*)data[i].sourcename);
+		splitnameandversion_done(&data[i].sourcename, &data[i].sourceversion);
 	}
 	return r;
 }

From 39d41e7d31b7c99a53c673d604b08857a238c664 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 12:19:42 +0100
Subject: [PATCH 26/70] [refactor] table_close: Set default return value

---
 database.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/database.c b/database.c
index ab67b62f..18107ded 100644
--- a/database.c
+++ b/database.c
@@ -995,7 +995,7 @@ static void table_printerror(struct table *table, int dbret, const char *action)
 
 retvalue table_close(struct table *table) {
 	int dbret;
-	retvalue result;
+	retvalue result = RET_OK;
 
 	if (table == NULL)
 		return RET_NOTHING;
@@ -1011,8 +1011,7 @@ retvalue table_close(struct table *table) {
 				table->name, table->subname,
 				db_strerror(dbret));
 		result = RET_DBERR(dbret);
-	} else
-		result = RET_OK;
+	}
 	free(table->name);
 	free(table->subname);
 	free(table);

From 235f3654d9d28fc63e02423d041996ec6cedf27d Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 10 Apr 2017 12:12:04 +0200
Subject: [PATCH 27/70] [refactor] target: Change error handling

Use variable 'result' only for the final returned result.
---
 target.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/target.c b/target.c
index 7937e131..dd993c24 100644
--- a/target.c
+++ b/target.c
@@ -347,7 +347,7 @@ retvalue package_remove_by_cursor(struct package_cursor *tc, struct logger *logg
 
 static retvalue addpackages(struct target *target, const char *packagename, const char *controlchunk, const char *version, const struct strlist *files, /*@null@*/const struct package *old, /*@null@*/const struct strlist *oldfiles, /*@null@*/struct logger *logger, /*@null@*/struct trackingdata *trackingdata, architecture_t architecture, /*@null@*/const char *causingrule, /*@null@*/const char *suitefrom) {
 
-	retvalue result, r;
+	retvalue result = RET_OK, r;
 	struct table *table = target->packages;
 	enum filetype filetype;
 
@@ -370,14 +370,14 @@ static retvalue addpackages(struct target *target, const char *packagename, cons
 	/* Add package to the distribution's database */
 
 	if (old != NULL) {
-		result = table_replacerecord(table, packagename, controlchunk);
+		r = table_replacerecord(table, packagename, controlchunk);
 
 	} else {
-		result = table_adduniqrecord(table, packagename, controlchunk);
+		r = table_adduniqrecord(table, packagename, controlchunk);
 	}
 
-	if (RET_WAS_ERROR(result))
-		return result;
+	if (RET_WAS_ERROR(r))
+		return r;
 
 	if (logger != NULL) {
 		logger_log(logger, target, packagename,

From 4c387466d8dd3fa62e4d35e29e88a984b4eee595 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 14:57:17 +0100
Subject: [PATCH 28/70] Add helper function debianversioncompare()

---
 database.c | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/database.c b/database.c
index 18107ded..76d2ddab 100644
--- a/database.c
+++ b/database.c
@@ -221,6 +221,7 @@ static const uint32_t types[dbt_COUNT] = {
 	DB_HASH
 };
 
+static int debianversioncompare(UNUSED(DB *db), const DBT *a, const DBT *b);
 #if DB_VERSION_MAJOR >= 6
 static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b, size_t *locp);
 #else
@@ -1741,6 +1742,45 @@ retvalue database_openreferences(void) {
 	return RET_OK;
 }
 
+static int debianversioncompare(UNUSED(DB *db), const DBT *a, const DBT *b) {
+	const char *a_version;
+	const char *b_version;
+	int versioncmp;
+	// There is no way to indicate an error to the caller
+	// Thus return -1 in case of an error
+	retvalue r = -1;
+
+	if (a->size == 0 || ((char*)a->data)[a->size-1] != '\0') {
+		fprintf(stderr, "Database value '%.*s' empty or not NULL terminated.\n", a->size, (char*)a->data);
+		return r;
+	}
+	if (b->size == 0 || ((char*)b->data)[b->size-1] != '\0') {
+		fprintf(stderr, "Database value '%.*s' empty or not NULL terminated.\n", b->size, (char*)b->data);
+		return r;
+	}
+
+	a_version = strchr(a->data, '|');
+	if (a_version == NULL) {
+		fprintf(stderr, "Database value '%s' malformed. It should be 'package|version'.\n", (char*)a->data);
+		return r;
+	}
+	a_version++;
+	b_version = strchr(b->data, '|');
+	if (b_version == NULL) {
+		fprintf(stderr, "Database value '%s' malformed. It should be 'package|version'.\n", (char*)b->data);
+		return r;
+	}
+	b_version++;
+
+	r = dpkgversions_cmp(a_version, b_version, &versioncmp);
+	if (RET_WAS_ERROR(r)) {
+		fprintf(stderr, "Parse errors processing versions.\n");
+		return r;
+	}
+
+	return -versioncmp;
+}
+
 /* only compare the first 0-terminated part of the data */
 static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b
 #if DB_VERSION_MAJOR >= 6

From e5b43d23c20f7826a594d960327ad11b64d0ae3f Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 14:57:35 +0100
Subject: [PATCH 29/70] Add helper function get_package_name()

---
 database.c | 19 +++++++++++++++++++
 error.h    |  1 +
 2 files changed, 20 insertions(+)

diff --git a/database.c b/database.c
index 76d2ddab..b6caa41c 100644
--- a/database.c
+++ b/database.c
@@ -1824,6 +1824,25 @@ retvalue database_opentracking(const char *codename, bool readonly, struct table
 	return RET_OK;
 }
 
+static int get_package_name(DB *secondary, const DBT *pkey, const DBT *pdata, DBT *skey) {
+	const char *separator;
+	size_t length;
+
+	separator = memchr(pkey->data, '|', pkey->size);
+	if (unlikely(separator == NULL)) {
+		return DB_MALFORMED_KEY;
+	}
+
+	length = (size_t)separator - (size_t)pkey->data;
+	skey->flags = DB_DBT_APPMALLOC;
+	skey->data = strndup(pkey->data, length);
+	if (FAILEDTOALLOC(skey->data)) {
+		return RET_ERROR_OOM;
+	}
+	skey->size = length + 1;
+	return 0;
+}
+
 retvalue database_openpackages(const char *identifier, bool readonly, struct table **table_p) {
 	struct table *table;
 	retvalue r;
diff --git a/error.h b/error.h
index 0d5a46d7..715b1a87 100644
--- a/error.h
+++ b/error.h
@@ -10,6 +10,7 @@ bool interrupted(void);
 /* retvalue is simply an int.
  * just named to show it follows the given semantics */
 /*@numabstract@*/ enum retvalue_enum {
+	DB_MALFORMED_KEY = -30001,
 	RET_ERROR_INCOMING_DENY = -13,
 	RET_ERROR_INTERNAL = -12,
 	RET_ERROR_BZ2 = -11,

From e0a4d0a8df6420fa987e27a535b6a0fef6d1d9b6 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 13:25:46 +0100
Subject: [PATCH 30/70] Add helper function package_primarykey()

---
 target.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/target.h b/target.h
index 05792b2d..2664b614 100644
--- a/target.h
+++ b/target.h
@@ -108,4 +108,18 @@ static inline bool target_matches(const struct target *t, const struct atomlist
 		return false;
 	return true;
 }
+
+static inline char *package_primarykey(const char *packagename, const char *version) {
+	char *key;
+
+	assert (packagename != NULL);
+	assert (version != NULL);
+	key = malloc(strlen(packagename) + 1 + strlen(version) + 1);
+	if (key != NULL) {
+		strcpy(key, packagename);
+		strcat(key, "|");
+		strcat(key, version);
+	}
+	return key;
+}
 #endif

From db3001e634286b9d0ac9aa6421f62c6bd74eb5e4 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 17:37:09 +0100
Subject: [PATCH 31/70] Add helper function cascade_strcmp()

---
 copypackages.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/copypackages.c b/copypackages.c
index 4598fd38..e562aab7 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -59,6 +59,18 @@ struct package_list {
 	/*@null@*/struct target_package_list *targets;
 };
 
+// cascade_strcmp compares the two strings s1 and s2. If the strings are equal, the strings
+// t1 and t2 are compared.
+static int cascade_strcmp(const char *s1, const char *s2, const char *t1, const char *t2) {
+	int result;
+
+	result = strcmp(s1, s2);
+	if (result == 0) {
+		result = strcmp(t1, t2);
+	}
+	return result;
+}
+
 static retvalue list_newpackage(struct package_list *list, struct target *target, const char *sourcename, const char *sourceversion, const char *packagename, const char *packageversion, /*@out@*/struct selectedpackage **package_p) {
 	struct target_package_list *t, **t_p;
 	struct selectedpackage *package, **p_p;

From c22deb0c4930275aa1378621bd3a58ba0d6b0a07 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 18:02:01 +0100
Subject: [PATCH 32/70] Add helper function strcmp2()

---
 globals.h | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/globals.h b/globals.h
index 945341dc..dca8bb03 100644
--- a/globals.h
+++ b/globals.h
@@ -1,6 +1,8 @@
 #ifndef REPREPRO_GLOBALS_H
 #define REPREPRO_GLOBALS_H
 
+#include <string.h>
+
 #ifdef AVOID_CHECKPROBLEMS
 # define bool _Bool
 # define true (1==1)
@@ -91,4 +93,19 @@ enum compression { c_none, c_gzip, c_bzip2, c_lzma, c_xz, c_lunzip, c_COUNT };
 #define nzNEW(num, type) ((type *)calloc(num, sizeof(type)))
 #define arrayinsert(type, array, position, length) ({type *__var = array; memmove(__var + (position) + 1, __var + (position), sizeof(type) * ((length) - (position)));})
 
+// strcmp2 behaves like strcmp, but allows both strings to be NULL
+inline int strcmp2(const char *s1, const char *s2) {
+	if (s1 == NULL || s2 == NULL) {
+		if (s1 == NULL && s2 == NULL) {
+			return 0;
+		} else if (s1 == NULL) {
+			return -1;
+		} else {
+			return 1;
+		}
+	} else {
+		return strcmp(s1, s2);
+	}
+}
+
 #endif

From fff3550beb89b524fee35092f7882429a376af04 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 18:28:06 +0100
Subject: [PATCH 33/70] Add helper function table_newduplicatecursor()

---
 database.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 database.h |  1 +
 2 files changed, 56 insertions(+)

diff --git a/database.c b/database.c
index b6caa41c..15d964c3 100644
--- a/database.c
+++ b/database.c
@@ -1419,6 +1419,61 @@ static inline retvalue parse_pair(struct table *table, DBT Key, DBT Data, /*@nul
 	return RET_OK;
 }
 
+retvalue table_newduplicatecursor(struct table *table, const char *key, long long skip, struct cursor **cursor_p, const char **key_p, const char **data_p, size_t *datalen_p) {
+	struct cursor *cursor;
+	int dbret;
+	DBT Key, Data;
+	retvalue r;
+
+	r = newcursor(table, DB_NEXT_DUP, &cursor);
+	if(!RET_IS_OK(r)) {
+		return r;
+	}
+	SETDBT(Key, key);
+	CLEARDBT(Data);
+	dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_SET);
+	if (dbret == DB_NOTFOUND || dbret == DB_KEYEMPTY) {
+		(void)cursor->cursor->c_close(cursor->cursor);
+		free(cursor);
+		return RET_NOTHING;
+	}
+	if (dbret != 0) {
+		table_printerror(table, dbret, "c_get(DB_SET)");
+		(void)cursor->cursor->c_close(cursor->cursor);
+		free(cursor);
+		return RET_DBERR(dbret);
+	}
+
+	while (skip > 0) {
+		CLEARDBT(Key);
+		CLEARDBT(Data);
+
+		dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, cursor->flags);
+		if (dbret == DB_NOTFOUND) {
+			(void)cursor->cursor->c_close(cursor->cursor);
+			free(cursor);
+			return RET_NOTHING;
+		}
+		if (dbret != 0) {
+			table_printerror(table, dbret, "c_get(DB_NEXT_DUP)");
+			(void)cursor->cursor->c_close(cursor->cursor);
+			free(cursor);
+			return RET_DBERR(dbret);
+		}
+
+		skip--;
+	}
+
+	r = parse_data(table, Key, Data, key_p, data_p, datalen_p);
+	if (RET_WAS_ERROR(r)) {
+		(void)cursor->cursor->c_close(cursor->cursor);
+		free(cursor);
+		return r;
+	}
+	*cursor_p = cursor;
+	return RET_OK;
+}
+
 retvalue table_newduplicatepairedcursor(struct table *table, const char *key, struct cursor **cursor_p, const char **value_p, const char **data_p, size_t *datalen_p) {
 	struct cursor *cursor;
 	int dbret;
diff --git a/database.h b/database.h
index 44293c4b..275fe710 100644
--- a/database.h
+++ b/database.h
@@ -48,6 +48,7 @@ retvalue table_checkrecord(struct table *, const char *key, const char *data);
 retvalue table_removerecord(struct table *, const char *key, const char *data);
 
 retvalue table_newglobalcursor(struct table *, /*@out@*/struct cursor **);
+retvalue table_newduplicatecursor(struct table *, const char *, long long, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue table_newduplicatepairedcursor(struct table *, const char *, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue table_newpairedcursor(struct table *, const char *, const char *, /*@out@*/struct cursor **, /*@out@*//*@null@*/const char **, /*@out@*//*@null@*/size_t *);
 bool cursor_nexttempdata(struct table *, struct cursor *, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);

From b7d433ff529440b0a7748168242a69407de007f9 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 28 Mar 2017 16:26:37 +0200
Subject: [PATCH 34/70] package_openiterator: Support opened databases

This change is a preparation for the package_openduplicateiterator()
function.
---
 package.h | 1 +
 target.c  | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/package.h b/package.h
index dee9df26..728a146c 100644
--- a/package.h
+++ b/package.h
@@ -68,6 +68,7 @@ struct package_cursor {
 	/*@temp@*/struct target *target;
 	struct cursor *cursor;
 	struct package current;
+	bool close_database;
 };
 
 retvalue package_openiterator(struct target *, bool /*readonly*/, /*@out@*/struct package_cursor *);
diff --git a/target.c b/target.c
index dd993c24..7bea4e65 100644
--- a/target.c
+++ b/target.c
@@ -943,6 +943,7 @@ retvalue package_openiterator(struct target *t, bool readonly, /*@out@*/struct p
 	retvalue r, r2;
 	struct cursor *c;
 
+	tc->close_database = t->packages == NULL;
 	r = target_initpackagesdb(t, readonly);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
@@ -978,8 +979,12 @@ retvalue package_closeiterator(struct package_cursor *tc) {
 
 	package_done(&tc->current);
 	result = cursor_close(tc->target->packages, tc->cursor);
-	r = target_closepackagesdb(tc->target);
-	RET_UPDATE(result, r);
+	if (tc->close_database) {
+		r = target_closepackagesdb(tc->target);
+		RET_UPDATE(result, r);
+	} else {
+		tc->target = NULL;
+	}
 	return result;
 }
 

From 2a51a8ab44d0c2de484cfaae36a1286cf47990e1 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 12:48:36 +0100
Subject: [PATCH 35/70] Add helper function package_openduplicateiterator()

---
 package.h |  1 +
 target.c  | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/package.h b/package.h
index 728a146c..28f12fd8 100644
--- a/package.h
+++ b/package.h
@@ -72,6 +72,7 @@ struct package_cursor {
 };
 
 retvalue package_openiterator(struct target *, bool /*readonly*/, /*@out@*/struct package_cursor *);
+retvalue package_openduplicateiterator(struct target *t, const char *name, long long, /*@out@*/struct package_cursor *tc);
 bool package_next(struct package_cursor *);
 retvalue package_closeiterator(struct package_cursor *);
 
diff --git a/target.c b/target.c
index 7bea4e65..0756e220 100644
--- a/target.c
+++ b/target.c
@@ -961,6 +961,34 @@ retvalue package_openiterator(struct target *t, bool readonly, /*@out@*/struct p
 	return RET_OK;
 }
 
+retvalue package_openduplicateiterator(struct target *t, const char *name, long long skip, /*@out@*/struct package_cursor *tc) {
+	retvalue r, r2;
+	struct cursor *c;
+
+	tc->close_database = t->packages == NULL;
+	if (tc->close_database) {
+		r = target_initpackagesdb(t, READONLY);
+		assert (r != RET_NOTHING);
+		if (RET_WAS_ERROR(r))
+			return r;
+	}
+
+	memset(&tc->current, 0, sizeof(tc->current));
+	r = table_newduplicatecursor(t->packages, name, skip, &c, &tc->current.name,
+	                             &tc->current.control, &tc->current.controllen);
+	if (!RET_IS_OK(r)) {
+		if (tc->close_database) {
+			r2 = target_closepackagesdb(t);
+			RET_ENDUPDATE(r, r2);
+		}
+		return r;
+	}
+	tc->current.target = t;
+	tc->target = t;
+	tc->cursor = c;
+	return RET_OK;
+}
+
 bool package_next(struct package_cursor *tc) {
 	bool success;
 	package_done(&tc->current);

From a1083a7bd421d59f16f406414404c304e93bb840 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 10 Apr 2017 14:45:53 +0200
Subject: [PATCH 36/70] [refactor] Pass distribution to tracking_done

For a later commit, pass the distribution to tracking_done.
---
 copypackages.c |  2 +-
 incoming.c     |  4 ++--
 main.c         | 30 +++++++++++++++---------------
 needbuild.c    |  2 +-
 tracking.c     | 10 +++++-----
 tracking.h     |  2 +-
 6 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index e562aab7..575c4b24 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -332,7 +332,7 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 		RET_UPDATE(into->status, r);
 		RET_ENDUPDATE(result, r);
 	}
-	r = tracking_done(tracks);
+	r = tracking_done(tracks, into);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
diff --git a/incoming.c b/incoming.c
index 05bf6127..60361dae 100644
--- a/incoming.c
+++ b/incoming.c
@@ -1853,7 +1853,7 @@ static retvalue candidate_add_into(const struct incoming *i, const struct candid
 		r = trackingdata_summon(tracks, c->source, c->sourceversion,
 				&trackingdata);
 		if (RET_WAS_ERROR(r)) {
-			(void)tracking_done(tracks);
+			(void)tracking_done(tracks, into);
 			return r;
 		}
 		if (into->trackingoptions.needsources) {
@@ -1939,7 +1939,7 @@ static retvalue candidate_add_into(const struct incoming *i, const struct candid
 		retvalue r2;
 		r2 = trackingdata_finish(tracks, &trackingdata);
 		RET_UPDATE(r, r2);
-		r2 = tracking_done(tracks);
+		r2 = tracking_done(tracks, into);
 		RET_ENDUPDATE(r, r2);
 	}
 	return r;
diff --git a/main.c b/main.c
index c30da24e..f22502a0 100644
--- a/main.c
+++ b/main.c
@@ -753,7 +753,7 @@ ACTION_D(y, n, y, remove) {
 		}
 		r = trackingdata_new(tracks, &trackingdata);
 		if (RET_WAS_ERROR(r)) {
-			(void)tracking_done(tracks);
+			(void)tracking_done(tracks, distribution);
 			return r;
 		}
 	}
@@ -789,7 +789,7 @@ ACTION_D(y, n, y, remove) {
 			trackingdata_done(&trackingdata);
 		else
 			trackingdata_finish(tracks, &trackingdata);
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, distribution);
 		RET_ENDUPDATE(result, r);
 	}
 	if (verbose >= 0 && !RET_WAS_ERROR(result) && remaining > 0) {
@@ -886,7 +886,7 @@ static retvalue remove_packages(struct distribution *distribution, struct remove
 				}
 			}
 		}
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, distribution);
 		RET_ENDUPDATE(result, r);
 		return result;
 	}
@@ -1026,7 +1026,7 @@ ACTION_D(y, n, y, removefilter) {
 		else {
 			r = trackingdata_new(tracks, &trackingdata);
 			if (RET_WAS_ERROR(r)) {
-				(void)tracking_done(tracks);
+				(void)tracking_done(tracks, distribution);
 				term_free(condition);
 				return r;
 			}
@@ -1041,7 +1041,7 @@ ACTION_D(y, n, y, removefilter) {
 			condition);
 	if (tracks != NULL) {
 		trackingdata_finish(tracks, &trackingdata);
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, distribution);
 		RET_ENDUPDATE(result, r);
 	}
 	term_free(condition);
@@ -1088,7 +1088,7 @@ ACTION_D(y, n, y, removematched) {
 		else {
 			r = trackingdata_new(tracks, &trackingdata);
 			if (RET_WAS_ERROR(r)) {
-				(void)tracking_done(tracks);
+				(void)tracking_done(tracks, distribution);
 				return r;
 			}
 		}
@@ -1102,7 +1102,7 @@ ACTION_D(y, n, y, removematched) {
 			(void*)argv[2]);
 	if (tracks != NULL) {
 		trackingdata_finish(tracks, &trackingdata);
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, distribution);
 		RET_ENDUPDATE(result, r);
 	}
 	return result;
@@ -2299,7 +2299,7 @@ ACTION_D(n, n, y, removetrack) {
 
 	result = tracking_remove(tracks, argv[2], argv[3]);
 
-	r = tracking_done(tracks);
+	r = tracking_done(tracks, distribution);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
@@ -2390,7 +2390,7 @@ ACTION_D(n, n, y, tidytracks) {
 		}
 		r = tracking_tidyall(tracks);
 		RET_UPDATE(result, r);
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, d);
 		RET_ENDUPDATE(result, r);
 		if (RET_WAS_ERROR(result))
 			break;
@@ -2426,7 +2426,7 @@ ACTION_B(n, n, y, dumptracks) {
 			continue;
 		r = tracking_printall(tracks);
 		RET_UPDATE(result, r);
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, d);
 		RET_ENDUPDATE(result, r);
 		if (RET_WAS_ERROR(result))
 			break;
@@ -2804,7 +2804,7 @@ ACTION_D(y, y, y, includedeb) {
 
 	distribution_unloadoverrides(distribution);
 
-	r = tracking_done(tracks);
+	r = tracking_done(tracks, distribution);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
@@ -2878,7 +2878,7 @@ ACTION_D(y, y, y, includedsc) {
 	logger_wait();
 
 	distribution_unloadoverrides(distribution);
-	r = tracking_done(tracks);
+	r = tracking_done(tracks, distribution);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
@@ -2933,7 +2933,7 @@ ACTION_D(y, y, y, include) {
 	}
 	result = distribution_loaduploaders(distribution);
 	if (RET_WAS_ERROR(result)) {
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, distribution);
 		RET_ENDUPDATE(result, r);
 		return result;
 	}
@@ -2945,7 +2945,7 @@ ACTION_D(y, y, y, include) {
 
 	distribution_unloadoverrides(distribution);
 	distribution_unloaduploaders(distribution);
-	r = tracking_done(tracks);
+	r = tracking_done(tracks, distribution);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
@@ -3792,7 +3792,7 @@ ACTION_D(y, n, y, flood) {
 		RET_UPDATE(distribution->status, result);
 
 	if (tracks != NULL) {
-		r = tracking_done(tracks);
+		r = tracking_done(tracks, distribution);
 		RET_ENDUPDATE(result, r);
 	}
 	return result;
diff --git a/needbuild.c b/needbuild.c
index 6b37f675..fa8801a4 100644
--- a/needbuild.c
+++ b/needbuild.c
@@ -297,7 +297,7 @@ retvalue find_needs_build(struct distribution *distribution, architecture_t arch
 			onlycomponents, architecture_source, pt_dsc,
 			check_source_needs_build, &d);
 
-	r = tracking_done(d.tracks);
+	r = tracking_done(d.tracks, distribution);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
diff --git a/tracking.c b/tracking.c
index 55ab89f8..054c3ba7 100644
--- a/tracking.c
+++ b/tracking.c
@@ -46,7 +46,7 @@ struct s_tracking {
 	struct trackingoptions options;
 };
 
-retvalue tracking_done(trackingdb db) {
+retvalue tracking_done(trackingdb db, struct distribution *distribution) {
 	retvalue r;
 
 	if (db == NULL)
@@ -568,7 +568,7 @@ retvalue tracking_rereference(struct distribution *distribution) {
 		return result;
 	r = tracking_recreatereferences(tracks);
 	RET_UPDATE(result, r);
-	r = tracking_done(tracks);
+	r = tracking_done(tracks, distribution);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
@@ -673,7 +673,7 @@ retvalue tracking_foreach_ro(struct distribution *d, tracking_foreach_ro_action
 
 	r = table_newglobalcursor(t->table, &cursor);
 	if (!RET_IS_OK(r)) {
-		(void)tracking_done(t);
+		(void)tracking_done(t, d);
 		return r;
 	}
 
@@ -691,7 +691,7 @@ retvalue tracking_foreach_ro(struct distribution *d, tracking_foreach_ro_action
 	}
 	r = cursor_close(t->table, cursor);
 	RET_ENDUPDATE(result, r);
-	r = tracking_done(t);
+	r = tracking_done(t, d);
 	RET_ENDUPDATE(result, r);
 	return result;
 }
@@ -1432,7 +1432,7 @@ retvalue tracking_retrack(struct distribution *d, bool needsretrack) {
 		/* now remove everything no longer needed */
 		r = tracking_tidyall(tracks);
 	}
-	rr = tracking_done(tracks);
+	rr = tracking_done(tracks, d);
 	RET_ENDUPDATE(r, rr);
 	return r;
 }
diff --git a/tracking.h b/tracking.h
index 082ab620..d868ff60 100644
--- a/tracking.h
+++ b/tracking.h
@@ -18,7 +18,7 @@ retvalue tracking_parse(struct distribution *, struct configiterator *);
 retvalue tracking_retrack(struct distribution *, bool /*evenifnotstale*/);
 
 retvalue tracking_initialize(/*@out@*/trackingdb *, const struct distribution *, bool readonly);
-retvalue tracking_done(trackingdb);
+retvalue tracking_done(trackingdb, struct distribution *distribution);
 retvalue tracking_listdistributions(/*@out@*/struct strlist *);
 retvalue tracking_drop(const char *);
 

From 03abe61fada0b822d74751d0d294b9f0e1fe6084 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 10 Apr 2017 13:52:10 +0200
Subject: [PATCH 37/70] Remember opened tracking databases

---
 distribution.h |  1 +
 tracking.c     | 10 +++++++++-
 tracking.h     |  2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/distribution.h b/distribution.h
index d03118c5..31105241 100644
--- a/distribution.h
+++ b/distribution.h
@@ -76,6 +76,7 @@ struct distribution {
 		bool keepsources;
 		bool embargoalls;
 		} trackingoptions;
+	trackingdb trackingdb;
 	/* what content files to generate */
 	struct contentsoptions contents;
 	struct atomlist contents_architectures,
diff --git a/tracking.c b/tracking.c
index 054c3ba7..572e4ffa 100644
--- a/tracking.c
+++ b/tracking.c
@@ -55,10 +55,17 @@ retvalue tracking_done(trackingdb db, struct distribution *distribution) {
 	r = table_close(db->table);
 	free(db->codename);
 	free(db);
+	if (distribution->trackingdb == NULL) {
+		fprintf(stderr,
+"Internal Error: Tracking database was closed, but corresponding entry in the distribution structure %s is missing.\n",
+		       distribution->codename);
+	} else {
+		distribution->trackingdb = NULL;
+	}
 	return r;
 }
 
-retvalue tracking_initialize(/*@out@*/trackingdb *db, const struct distribution *distribution, bool readonly) {
+retvalue tracking_initialize(/*@out@*/trackingdb *db, struct distribution *distribution, bool readonly) {
 	struct s_tracking *t;
 	retvalue r;
 
@@ -80,6 +87,7 @@ retvalue tracking_initialize(/*@out@*/trackingdb *db, const struct distribution
 		return r;
 	}
 	*db = t;
+	distribution->trackingdb = t;
 	return RET_OK;
 }
 
diff --git a/tracking.h b/tracking.h
index d868ff60..e44383ef 100644
--- a/tracking.h
+++ b/tracking.h
@@ -17,7 +17,7 @@ retvalue tracking_parse(struct distribution *, struct configiterator *);
 /* high-level retrack of the whole distribution */
 retvalue tracking_retrack(struct distribution *, bool /*evenifnotstale*/);
 
-retvalue tracking_initialize(/*@out@*/trackingdb *, const struct distribution *, bool readonly);
+retvalue tracking_initialize(/*@out@*/trackingdb *, struct distribution *, bool readonly);
 retvalue tracking_done(trackingdb, struct distribution *distribution);
 retvalue tracking_listdistributions(/*@out@*/struct strlist *);
 retvalue tracking_drop(const char *);

From 762ed362c814ef0776a43b27441856bc200d3ec4 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 3 Feb 2017 12:03:47 +0100
Subject: [PATCH 38/70] Support listing multiple versions in ls command

Currently only one package version is supported for each target, but
prepare support for multiple versions. Instead of querying only one
package for each target in the ls command, iterate over all packages
with the given name for each target.
---
 main.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/main.c b/main.c
index f22502a0..2919310e 100644
--- a/main.c
+++ b/main.c
@@ -1295,17 +1295,20 @@ static retvalue newlsversion(struct lsversion **versions_p, struct package *pack
 
 static retvalue ls_in_target(struct target *target, const char *packagename, struct lsversion **versions_p) {
 	retvalue r, result;
-	struct package pkg;
+	struct package_cursor iterator;
 
-	result = package_get(target, packagename, NULL, &pkg);
-	if (RET_IS_OK(result)) {
-		r = package_getversion(&pkg);
+	result = package_openduplicateiterator(target, packagename, 0, &iterator);
+	if (!RET_IS_OK(result))
+		return result;
+	do {
+		r = package_getversion(&iterator.current);
 		if (RET_IS_OK(r))
-			r = newlsversion(versions_p, &pkg,
+			r = newlsversion(versions_p, &iterator.current,
 					target->architecture);
-		package_done(&pkg);
 		RET_UPDATE(result, r);
-	}
+	} while (package_next(&iterator));
+	r = package_closeiterator(&iterator);
+	RET_ENDUPDATE(result, r);
 	return result;
 }
 

From 8236dc7aaff23ad7d37bdb446a58b56a3cf03fd8 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 8 Feb 2017 11:38:02 +0100
Subject: [PATCH 39/70] Support listing multiple versions in list command

Currently only one package version is supported for each target, but
prepare support for multiple versions. Instead of querying only one
package for each target in the list command, iterate over all packages
with the given name for each target.
---
 main.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/main.c b/main.c
index 2919310e..bba9180a 100644
--- a/main.c
+++ b/main.c
@@ -1202,23 +1202,27 @@ ACTION_C(n, n, n, listcodenames) {
 }
 
 static retvalue list_in_target(struct target *target, const char *packagename) {
-	retvalue r, result;
-	struct package pkg;
+	retvalue r, result = RET_NOTHING;
+	struct package_cursor iterator;
 
 	if (listmax == 0)
 		return RET_NOTHING;
 
-	result = package_get(target, packagename, NULL, &pkg);
-	if (RET_IS_OK(result)) {
+	r = package_openduplicateiterator(target, packagename, 0, &iterator);
+	if (!RET_IS_OK(r))
+		return r;
+
+	do {
 		if (listskip <= 0) {
-			r = listformat_print(listformat, &pkg);
+			r = listformat_print(listformat, &iterator.current);
 			RET_UPDATE(result, r);
 			if (listmax > 0)
 				listmax--;
 		} else
 			listskip--;
-		package_done(&pkg);
-	}
+	} while (package_next(&iterator));
+	r = package_closeiterator(&iterator);
+	RET_ENDUPDATE(result, r);
 	return result;
 }
 

From fd431654245071b62accb8213603bf55248f24a9 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 29 Aug 2018 12:37:01 +0200
Subject: [PATCH 40/70] [refactor] Add duplicate option to
 table_newglobalcursor

Allow to open a cursor that either iterates over all database entries or
only over the first of each duplicate (i.e. only the latest version of
each package).
---
 database.c  | 12 ++++++------
 database.h  |  2 +-
 filelist.c  |  2 +-
 files.c     | 10 +++++-----
 main.c      |  2 +-
 reference.c |  4 ++--
 sizes.c     |  2 +-
 target.c    |  2 +-
 tracking.c  | 10 +++++-----
 9 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/database.c b/database.c
index 15d964c3..d0dca04d 100644
--- a/database.c
+++ b/database.c
@@ -1350,10 +1350,10 @@ static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **c
 	return RET_OK;
 }
 
-retvalue table_newglobalcursor(struct table *table, struct cursor **cursor_p) {
+retvalue table_newglobalcursor(struct table *table, bool duplicate, struct cursor **cursor_p) {
 	retvalue r;
 
-	r = newcursor(table, DB_NEXT, cursor_p);
+	r = newcursor(table, duplicate ? DB_NEXT : DB_NEXT_NODUP, cursor_p);
 	if (r == RET_NOTHING) {
 		// table_newglobalcursor returned RET_OK when table->berkeleydb == NULL. Is that return value wanted?
 		r = RET_OK;
@@ -2055,7 +2055,7 @@ static retvalue table_copy(struct table *oldtable, struct table *newtable) {
 	const char *filekey, *data;
 	size_t data_len;
 
-	r = table_newglobalcursor(oldtable, &cursor);
+	r = table_newglobalcursor(oldtable, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	while (cursor_nexttempdata(oldtable, cursor, &filekey,
@@ -2179,7 +2179,7 @@ static inline retvalue translate(struct table *oldmd5sums, struct table *newchec
 
 	/* first add all md5sums to checksums if not there yet */
 
-	r = table_newglobalcursor(oldmd5sums, &cursor);
+	r = table_newglobalcursor(oldmd5sums, true, &cursor);
 	if (RET_WAS_ERROR(r))
 		return r;
 	while (cursor_nexttempdata(oldmd5sums, cursor,
@@ -2242,10 +2242,10 @@ static inline retvalue translate(struct table *oldmd5sums, struct table *newchec
 
 	/* then delete everything from checksums that is not in md5sums */
 
-	r = table_newglobalcursor(oldmd5sums, &cursor);
+	r = table_newglobalcursor(oldmd5sums, true, &cursor);
 	if (RET_WAS_ERROR(r))
 		return r;
-	r = table_newglobalcursor(newchecksums, &newcursor);
+	r = table_newglobalcursor(newchecksums, true, &newcursor);
 	if (RET_WAS_ERROR(r)) {
 		cursor_close(oldmd5sums, cursor);
 		return r;
diff --git a/database.h b/database.h
index 275fe710..25bc3ca0 100644
--- a/database.h
+++ b/database.h
@@ -47,7 +47,7 @@ retvalue table_deleterecord(struct table *, const char *key, bool ignoremissing)
 retvalue table_checkrecord(struct table *, const char *key, const char *data);
 retvalue table_removerecord(struct table *, const char *key, const char *data);
 
-retvalue table_newglobalcursor(struct table *, /*@out@*/struct cursor **);
+retvalue table_newglobalcursor(struct table *, bool /*duplicate*/, /*@out@*/struct cursor **);
 retvalue table_newduplicatecursor(struct table *, const char *, long long, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue table_newduplicatepairedcursor(struct table *, const char *, /*@out@*/struct cursor **, /*@out@*/const char **, /*@out@*/const char **, /*@out@*/size_t *);
 retvalue table_newpairedcursor(struct table *, const char *, const char *, /*@out@*/struct cursor **, /*@out@*//*@null@*/const char **, /*@out@*//*@null@*/size_t *);
diff --git a/filelist.c b/filelist.c
index 5c39da66..1cb7258d 100644
--- a/filelist.c
+++ b/filelist.c
@@ -693,7 +693,7 @@ retvalue filelists_translate(struct table *oldtable, struct table *newtable) {
 	size_t olddata_len, newdata_size;
 	char *newdata;
 
-	r = table_newglobalcursor(oldtable, &cursor);
+	r = table_newglobalcursor(oldtable, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	while (cursor_nexttempdata(oldtable, cursor, &filekey,
diff --git a/files.c b/files.c
index f378b70a..0f3b6180 100644
--- a/files.c
+++ b/files.c
@@ -276,7 +276,7 @@ retvalue files_printmd5sums(void) {
 	struct cursor *cursor;
 	const char *filekey, *checksum;
 
-	r = table_newglobalcursor(rdb_checksums, &cursor);
+	r = table_newglobalcursor(rdb_checksums, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
@@ -303,7 +303,7 @@ retvalue files_printchecksums(void) {
 	struct cursor *cursor;
 	const char *filekey, *checksum;
 
-	r = table_newglobalcursor(rdb_checksums, &cursor);
+	r = table_newglobalcursor(rdb_checksums, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
@@ -329,7 +329,7 @@ retvalue files_foreach(per_file_action action, void *privdata) {
 	struct cursor *cursor;
 	const char *filekey, *checksum;
 
-	r = table_newglobalcursor(rdb_checksums, &cursor);
+	r = table_newglobalcursor(rdb_checksums, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
@@ -375,7 +375,7 @@ retvalue files_checkpool(bool fast) {
 	bool improveable = false;
 
 	result = RET_NOTHING;
-	r = table_newglobalcursor(rdb_checksums, &cursor);
+	r = table_newglobalcursor(rdb_checksums, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	while (cursor_nexttempdata(rdb_checksums, cursor,
@@ -421,7 +421,7 @@ retvalue files_collectnewchecksums(void) {
 	char *fullfilename;
 
 	result = RET_NOTHING;
-	r = table_newglobalcursor(rdb_checksums, &cursor);
+	r = table_newglobalcursor(rdb_checksums, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 	while (cursor_nexttempdata(rdb_checksums, cursor,
diff --git a/main.c b/main.c
index bba9180a..c2a24e18 100644
--- a/main.c
+++ b/main.c
@@ -1594,7 +1594,7 @@ ACTION_B(n, n, n, dumpcontents) {
 	result = database_openpackages(argv[1], true, &packages);
 	if (RET_WAS_ERROR(result))
 		return result;
-	r = table_newglobalcursor(packages, &cursor);
+	r = table_newglobalcursor(packages, true, &cursor);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r)) {
 		(void)table_close(packages);
diff --git a/reference.c b/reference.c
index 563cfe76..5f59e9f4 100644
--- a/reference.c
+++ b/reference.c
@@ -152,7 +152,7 @@ retvalue references_remove(const char *neededby) {
 	const char *found_to, *found_by;
 	size_t datalen, l;
 
-	r = table_newglobalcursor(rdb_references, &cursor);
+	r = table_newglobalcursor(rdb_references, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 
@@ -188,7 +188,7 @@ retvalue references_dump(void) {
 	retvalue result, r;
 	const char *found_to, *found_by;
 
-	r = table_newglobalcursor(rdb_references, &cursor);
+	r = table_newglobalcursor(rdb_references, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 
diff --git a/sizes.c b/sizes.c
index 34f76710..b10ccab8 100644
--- a/sizes.c
+++ b/sizes.c
@@ -225,7 +225,7 @@ retvalue sizes_distributions(struct distribution *alldistributions, bool specifi
 	}
 	if (ds == NULL)
 		return RET_NOTHING;
-	r = table_newglobalcursor(rdb_references, &cursor);
+	r = table_newglobalcursor(rdb_references, true, &cursor);
 	if (!RET_IS_OK(r)) {
 		distribution_sizes_freelist(ds);
 		return r;
diff --git a/target.c b/target.c
index 0756e220..64d2353d 100644
--- a/target.c
+++ b/target.c
@@ -948,7 +948,7 @@ retvalue package_openiterator(struct target *t, bool readonly, /*@out@*/struct p
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
 		return r;
-	r = table_newglobalcursor(t->packages, &c);
+	r = table_newglobalcursor(t->packages, true, &c);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r)) {
 		r2 = target_closepackagesdb(t);
diff --git a/tracking.c b/tracking.c
index 572e4ffa..1d3cef5d 100644
--- a/tracking.c
+++ b/tracking.c
@@ -530,7 +530,7 @@ static retvalue tracking_recreatereferences(trackingdb t) {
 	const char *key, *value, *data;
 	size_t datalen;
 
-	r = table_newglobalcursor(t->table, &cursor);
+	r = table_newglobalcursor(t->table, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 
@@ -647,7 +647,7 @@ retvalue tracking_printall(trackingdb t) {
 	const char *key, *value, *data;
 	size_t datalen;
 
-	r = table_newglobalcursor(t->table, &cursor);
+	r = table_newglobalcursor(t->table, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 
@@ -679,7 +679,7 @@ retvalue tracking_foreach_ro(struct distribution *d, tracking_foreach_ro_action
 	if (!RET_IS_OK(r))
 		return r;
 
-	r = table_newglobalcursor(t->table, &cursor);
+	r = table_newglobalcursor(t->table, true, &cursor);
 	if (!RET_IS_OK(r)) {
 		(void)tracking_done(t, d);
 		return r;
@@ -1077,7 +1077,7 @@ retvalue tracking_tidyall(trackingdb t) {
 	const char *key, *value, *data;
 	size_t datalen;
 
-	r = table_newglobalcursor(t->table, &cursor);
+	r = table_newglobalcursor(t->table, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 
@@ -1110,7 +1110,7 @@ retvalue tracking_reset(trackingdb t) {
 	size_t datalen, newdatalen;
 	int i;
 
-	r = table_newglobalcursor(t->table, &cursor);
+	r = table_newglobalcursor(t->table, true, &cursor);
 	if (!RET_IS_OK(r))
 		return r;
 

From 9190cbc4a80cf278c3c51ad24409e6cdfa099623 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 29 Aug 2018 12:48:04 +0200
Subject: [PATCH 41/70] [refactor] package_openiterator: Pass through duplicate
 option

---
 archallflood.c |  4 ++--
 contents.c     |  2 +-
 copypackages.c |  6 +++---
 distribution.c |  6 +++---
 exports.c      |  2 +-
 main.c         |  2 +-
 package.h      |  2 +-
 sourcecheck.c  |  4 ++--
 target.c       | 10 +++++-----
 upgradelist.c  |  4 ++--
 10 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/archallflood.c b/archallflood.c
index 4c7a43bc..1a373394 100644
--- a/archallflood.c
+++ b/archallflood.c
@@ -327,7 +327,7 @@ static retvalue floodlist_initialize(struct floodlist **fl, struct target *t) {
 
 	/* Begin with the packages currently in the archive */
 
-	r = package_openiterator(t, READONLY, &iterator);
+	r = package_openiterator(t, READONLY, true, &iterator);
 	if (RET_WAS_ERROR(r)) {
 		floodlist_free(list);
 		return r;
@@ -549,7 +549,7 @@ static retvalue floodlist_pull(struct floodlist *list, struct target *source) {
 	struct package_cursor iterator;
 
 	list->last = NULL;
-	r = package_openiterator(source, READONLY, &iterator);
+	r = package_openiterator(source, READONLY, true, &iterator);
 	if (RET_WAS_ERROR(r))
 		return r;
 	result = RET_NOTHING;
diff --git a/contents.c b/contents.c
index 495fb234..40da6bc8 100644
--- a/contents.c
+++ b/contents.c
@@ -215,7 +215,7 @@ static retvalue gentargetcontents(struct target *target, struct release *release
 		release_abortfile(file);
 		return r;
 	}
-	result = package_openiterator(target, READONLY, &iterator);
+	result = package_openiterator(target, READONLY, true, &iterator);
 	if (RET_IS_OK(result)) {
 		while (package_next(&iterator)) {
 			r = addpackagetocontents(&iterator.current, contents);
diff --git a/copypackages.c b/copypackages.c
index 575c4b24..9663158e 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -483,7 +483,7 @@ static retvalue by_source(struct package_list *list, struct target *desttarget,
 
 	assert (d->argc > 0);
 
-	r = package_openiterator(fromtarget, READONLY, &iterator);
+	r = package_openiterator(fromtarget, READONLY, true, &iterator);
 	assert (r != RET_NOTHING);
 	if (!RET_IS_OK(r))
 		return r;
@@ -621,7 +621,7 @@ static retvalue by_formula(struct package_list *list, struct target *desttarget,
 	struct package_cursor iterator;
 	retvalue result, r;
 
-	r = package_openiterator(fromtarget, READONLY, &iterator);
+	r = package_openiterator(fromtarget, READONLY, true, &iterator);
 	assert (r != RET_NOTHING);
 	if (!RET_IS_OK(r))
 		return r;
@@ -650,7 +650,7 @@ static retvalue by_glob(struct package_list *list, struct target *desttarget, st
 	struct package_cursor iterator;
 	retvalue result, r;
 
-	r = package_openiterator(fromtarget, READONLY, &iterator);
+	r = package_openiterator(fromtarget, READONLY, true, &iterator);
 	assert (r != RET_NOTHING);
 	if (!RET_IS_OK(r))
 		return r;
diff --git a/distribution.c b/distribution.c
index cb98055d..f7144d61 100644
--- a/distribution.c
+++ b/distribution.c
@@ -552,7 +552,7 @@ retvalue package_foreach(struct distribution *distribution, const struct atomlis
 			if (r == RET_NOTHING)
 				continue;
 		}
-		r = package_openiterator(t, READONLY, &iterator);
+		r = package_openiterator(t, READONLY, true, &iterator);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			return result;
@@ -584,7 +584,7 @@ retvalue package_foreach_c(struct distribution *distribution, const struct atoml
 			continue;
 		if (limitation_missed(packagetype, t->packagetype))
 			continue;
-		r = package_openiterator(t, READONLY, &iterator);
+		r = package_openiterator(t, READONLY, true, &iterator);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			return result;
@@ -1120,7 +1120,7 @@ retvalue package_remove_each(struct distribution *distribution, const struct ato
 	for (t = distribution->targets ; t != NULL ; t = t->next) {
 		if (!target_matches(t, components, architectures, packagetypes))
 			continue;
-		r = package_openiterator(t, READWRITE, &iterator);
+		r = package_openiterator(t, READWRITE, true, &iterator);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			return result;
diff --git a/exports.c b/exports.c
index 0bc86d27..aef8a855 100644
--- a/exports.c
+++ b/exports.c
@@ -490,7 +490,7 @@ retvalue export_target(const char *relativedir, struct target *target,  const st
 					exportdescription(exportmode, buffer, 100));
 			status = "new";
 		}
-		r = package_openiterator(target, READONLY, &iterator);
+		r = package_openiterator(target, READONLY, true, &iterator);
 		if (RET_WAS_ERROR(r)) {
 			release_abortfile(file);
 			free(relfilename);
diff --git a/main.c b/main.c
index c2a24e18..54840529 100644
--- a/main.c
+++ b/main.c
@@ -2556,7 +2556,7 @@ static retvalue repair_descriptions(struct target *target) {
 				target->identifier);
 	}
 
-	r = package_openiterator(target, READWRITE, &iterator);
+	r = package_openiterator(target, READWRITE, true, &iterator);
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
diff --git a/package.h b/package.h
index 28f12fd8..7a46a04c 100644
--- a/package.h
+++ b/package.h
@@ -71,7 +71,7 @@ struct package_cursor {
 	bool close_database;
 };
 
-retvalue package_openiterator(struct target *, bool /*readonly*/, /*@out@*/struct package_cursor *);
+retvalue package_openiterator(struct target *, bool /*readonly*/, bool /*duplicate*/, /*@out@*/struct package_cursor *);
 retvalue package_openduplicateiterator(struct target *t, const char *name, long long, /*@out@*/struct package_cursor *tc);
 bool package_next(struct package_cursor *);
 retvalue package_closeiterator(struct package_cursor *);
diff --git a/sourcecheck.c b/sourcecheck.c
index 0c23176a..8182639f 100644
--- a/sourcecheck.c
+++ b/sourcecheck.c
@@ -76,7 +76,7 @@ static retvalue collect_source_versions(struct distribution *d, struct info_sour
 	for (t = d->targets ; t != NULL ; t = t->next) {
 		if (t->architecture != architecture_source)
 			continue;
-		r = package_openiterator(t, true, &cursor);
+		r = package_openiterator(t, true, true, &cursor);
 		if (RET_WAS_ERROR(r)) {
 			RET_UPDATE(result, r);
 			break;
@@ -196,7 +196,7 @@ static retvalue process_binaries(struct distribution *d, struct info_source *sou
 	for (t = d->targets ; t != NULL ; t = t->next) {
 		if (t->architecture == architecture_source)
 			continue;
-		r = package_openiterator(t, true, &cursor);
+		r = package_openiterator(t, true, true, &cursor);
 		if (RET_WAS_ERROR(r)) {
 			RET_UPDATE(result, r);
 			break;
diff --git a/target.c b/target.c
index 64d2353d..584bac23 100644
--- a/target.c
+++ b/target.c
@@ -629,7 +629,7 @@ retvalue target_rereference(struct target *target) {
 	if (verbose > 2)
 		printf("Referencing %s...\n", target->identifier);
 
-	r = package_openiterator(target, READONLY, &iterator);
+	r = package_openiterator(target, READONLY, true, &iterator);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
 		return r;
@@ -753,7 +753,7 @@ retvalue target_reoverride(struct target *target, struct distribution *distribut
 				target->identifier);
 	}
 
-	r = package_openiterator(target, READWRITE, &iterator);
+	r = package_openiterator(target, READWRITE, true, &iterator);
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
@@ -821,7 +821,7 @@ retvalue target_redochecksums(struct target *target, struct distribution *distri
 				target->identifier);
 	}
 
-	r = package_openiterator(target, READWRITE, &iterator);
+	r = package_openiterator(target, READWRITE, true, &iterator);
 	if (!RET_IS_OK(r))
 		return r;
 	result = RET_NOTHING;
@@ -939,7 +939,7 @@ retvalue package_get(struct target *target, const char *name, const char *versio
 	return result;
 }
 
-retvalue package_openiterator(struct target *t, bool readonly, /*@out@*/struct package_cursor *tc) {
+retvalue package_openiterator(struct target *t, bool readonly, bool duplicate, /*@out@*/struct package_cursor *tc) {
 	retvalue r, r2;
 	struct cursor *c;
 
@@ -948,7 +948,7 @@ retvalue package_openiterator(struct target *t, bool readonly, /*@out@*/struct p
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
 		return r;
-	r = table_newglobalcursor(t->packages, true, &c);
+	r = table_newglobalcursor(t->packages, duplicate, &c);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r)) {
 		r2 = target_closepackagesdb(t);
diff --git a/upgradelist.c b/upgradelist.c
index 41995c80..c6c73814 100644
--- a/upgradelist.c
+++ b/upgradelist.c
@@ -151,7 +151,7 @@ retvalue upgradelist_initialize(struct upgradelist **ul, struct target *t) {
 
 	/* Beginn with the packages currently in the archive */
 
-	r = package_openiterator(t, READONLY, &iterator);
+	r = package_openiterator(t, READONLY, true, &iterator);
 	if (RET_WAS_ERROR(r)) {
 		upgradelist_free(upgrade);
 		return r;
@@ -519,7 +519,7 @@ retvalue upgradelist_pull(struct upgradelist *upgrade, struct target *source, up
 	struct package_cursor iterator;
 
 	upgrade->last = NULL;
-	r = package_openiterator(source, READONLY, &iterator);
+	r = package_openiterator(source, READONLY, true, &iterator);
 	if (RET_WAS_ERROR(r))
 		return r;
 	result = RET_NOTHING;

From 00ddf0a88b7c05409441366f3618675ab55ed0ea Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 29 Aug 2018 13:29:15 +0200
Subject: [PATCH 42/70] Fix "Package database is not sorted" in update command

When multiple versions of one package are available in the archive, the
update command will fail:

```
Calculating packages to get...
Package database is not sorted!!!
reprepro: upgradelist.c:135: save_package_version: Assertion `false'
failed.
Aborted
```

Fix this assertion error by iterating only over the newest version of
each package.

fixes #6
---
 upgradelist.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/upgradelist.c b/upgradelist.c
index c6c73814..a02c44ff 100644
--- a/upgradelist.c
+++ b/upgradelist.c
@@ -151,7 +151,7 @@ retvalue upgradelist_initialize(struct upgradelist **ul, struct target *t) {
 
 	/* Beginn with the packages currently in the archive */
 
-	r = package_openiterator(t, READONLY, true, &iterator);
+	r = package_openiterator(t, READONLY, false, &iterator);
 	if (RET_WAS_ERROR(r)) {
 		upgradelist_free(upgrade);
 		return r;

From 3959181e00fd3ccb2e04e88d60993ca5c4c8bb27 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 2 Feb 2017 19:05:42 +0100
Subject: [PATCH 43/70] [refactor] Introduce nameandversion struct for copy
 command

The multiple version support will require to pass a list of names and
versions to the copy_by_name() function instead of just a list of names.
Thus introduce a nameandversion struct that also holds the data needed
for the copy_by_name() function.
---
 copypackages.c | 63 ++++++++++++++++++++++++++------------------------
 copypackages.h |  9 +++++++-
 main.c         | 12 +++++++++-
 3 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index 9663158e..1afc5706 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -375,30 +375,37 @@ struct namelist {
 };
 
 static retvalue by_name(struct package_list *list, struct target *desttarget, struct target *fromtarget, void *data) {
-	struct namelist *d = data;
+	struct nameandversion *nameandversion = data;
+	struct nameandversion *prev;
 	retvalue result, r;
-	int i, j;
 
 	result = RET_NOTHING;
-	for (i = 0 ; i < d->argc ; i++) {
+	for (struct nameandversion *d = nameandversion; d->name != NULL ; d++) {
 		struct package package;
-		const char *name = d->argv[i];
 
-		for (j = 0 ; j < i ; j++)
-			if (strcmp(d->argv[i], d->argv[j]) == 0)
+		for (prev = nameandversion ; prev < d ; prev++) {
+			if (strcmp(prev->name, d->name) == 0 && strcmp2(prev->version, d->version) == 0)
 				break;
-		if (j < i) {
-			if (verbose >= 0 && ! d->warnedabout[j])
-				fprintf(stderr,
+		}
+		if (prev < d) {
+			if (verbose >= 0 && ! prev->warnedabout) {
+				if (d->version == NULL) {
+					fprintf(stderr,
 "Hint: '%s' was listed multiple times, ignoring all but first!\n",
-						d->argv[i]);
-			d->warnedabout[j] = true;
+							d->name);
+				} else {
+					fprintf(stderr,
+"Hint: '%s=%s' was listed multiple times, ignoring all but first!\n",
+							d->name, d->version);
+				}
+			}
+			prev->warnedabout = true;
 			/* do not complain second is missing if we ignore it: */
-			d->found[i] = true;
+			d->found = true;
 			continue;
 		}
 
-		r = package_get(fromtarget, name, NULL, &package);
+		r = package_get(fromtarget, d->name, d->version, &package);
 		if (r == RET_NOTHING)
 			continue;
 		RET_ENDUPDATE(result, r);
@@ -409,7 +416,7 @@ static retvalue by_name(struct package_list *list, struct target *desttarget, st
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			break;
-		d->found[i] = true;
+		d->found = true;
 	}
 	return result;
 }
@@ -430,30 +437,23 @@ static void packagelist_done(struct package_list *list) {
 	}
 }
 
-retvalue copy_by_name(struct distribution *into, struct distribution *from, int argc, const char **argv, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes) {
+retvalue copy_by_name(struct distribution *into, struct distribution *from, struct nameandversion *nameandversion, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes) {
 	struct package_list list;
-	struct namelist names = {
-		argc, argv, nzNEW(argc, bool), nzNEW(argc, bool)
-	};
 	retvalue r;
 
-	if (FAILEDTOALLOC(names.warnedabout) || FAILEDTOALLOC(names.found)) {
-		free(names.found);
-		free(names.warnedabout);
-		return RET_ERROR_OOM;
+	for (struct nameandversion *d = nameandversion; d->name != NULL; d++) {
+		d->found = false;
+		d->warnedabout = false;
 	}
 
 	memset(&list, 0, sizeof(list));
 	r = copy_by_func(&list, into, from, components,
-			architectures, packagetypes, by_name, &names);
-	free(names.warnedabout);
+			architectures, packagetypes, by_name, nameandversion);
 	if (verbose >= 0 && !RET_WAS_ERROR(r)) {
-		int i;
 		bool first = true;
 
-		assert(names.found != NULL);
-		for (i = 0 ; i < argc ; i++) {
-			if (names.found[i])
+		for (struct nameandversion *d = nameandversion; d->name != NULL; d++) {
+			if (d->found)
 				continue;
 			if (first)
 				(void)fputs(
@@ -461,14 +461,17 @@ retvalue copy_by_name(struct distribution *into, struct distribution *from, int
 			else
 				(void)fputs(", ", stderr);
 			first = false;
-			(void)fputs(argv[i], stderr);
+			(void)fputs(d->name, stderr);
+			if (d->version != NULL) {
+				(void)fputs("=", stderr);
+				(void)fputs(d->version, stderr);
+			}
 		}
 		if (!first) {
 			(void)fputc('.', stderr);
 			(void)fputc('\n', stderr);
 		}
 	}
-	free(names.found);
 	if (!RET_IS_OK(r))
 		return r;
 	r = packagelist_add(into, &list, from->codename);
diff --git a/copypackages.h b/copypackages.h
index 3515bcd4..57682065 100644
--- a/copypackages.h
+++ b/copypackages.h
@@ -5,7 +5,14 @@
 #include "strlist.h"
 #endif
 
-retvalue copy_by_name(struct distribution * /*into*/, struct distribution * /*from*/, int, const char **, const struct atomlist *, const struct atomlist *, const struct atomlist *);
+struct nameandversion {
+	const char *name;
+	const char /*@null@*/ *version;
+	bool warnedabout;
+	bool found;
+};
+
+retvalue copy_by_name(struct distribution * /*into*/, struct distribution * /*from*/, struct nameandversion *, const struct atomlist *, const struct atomlist *, const struct atomlist *);
 retvalue copy_by_source(struct distribution * /*into*/, struct distribution * /*from*/, int, const char **, const struct atomlist *, const struct atomlist *, const struct atomlist *);
 retvalue copy_by_formula(struct distribution * /*into*/, struct distribution * /*from*/, const char *formula, const struct atomlist *, const struct atomlist *, const struct atomlist *);
 retvalue copy_by_glob(struct distribution * /*into*/, struct distribution * /*from*/, const char * /*glob*/, const struct atomlist *, const struct atomlist *, const struct atomlist *);
diff --git a/main.c b/main.c
index 54840529..701870ec 100644
--- a/main.c
+++ b/main.c
@@ -1957,6 +1957,8 @@ ACTION_B(y, n, y, dumppull) {
 
 ACTION_D(y, n, y, copy) {
 	struct distribution *destination, *source;
+	struct nameandversion data[argc-2];
+	int i;
 	retvalue result;
 
 	result = distribution_get(alldistributions, argv[1], true, &destination);
@@ -1978,8 +1980,16 @@ ACTION_D(y, n, y, copy) {
 	if (RET_WAS_ERROR(result))
 		return result;
 
-	return copy_by_name(destination, source, argc-3, argv+3,
+	for (i = 0; i < argc-3; i++) {
+		data[i].name = argv[3 + i];
+		data[i].version = NULL;
+	}
+	data[i].name = NULL;
+	data[i].version = NULL;
+
+	result = copy_by_name(destination, source, data,
 			components, architectures, packagetypes);
+	return result;
 }
 
 ACTION_D(y, n, y, copysrc) {

From 6ed0523617827b5c80ba307c4392e73f94d67509 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 12:39:44 +0100
Subject: [PATCH 44/70] [refactor] Use nameandversion struct for remove command

The multiple version support will require to pass a list of names and
versions to the remove_from_target() function instead of just a list of
names. Thus use the nameandversion struct for the remove command.

Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 main.c | 43 ++++++++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/main.c b/main.c
index 701870ec..ae55caa0 100644
--- a/main.c
+++ b/main.c
@@ -701,19 +701,19 @@ ACTION_R(n, n, n, y, addreferences) {
 	return ret;
 }
 
-static retvalue remove_from_target(struct distribution *distribution, struct trackingdata *trackingdata, struct target *target, int count, const char * const *names, int *remaining, bool *gotremoved) {
+static retvalue remove_from_target(struct distribution *distribution, struct trackingdata *trackingdata, struct target *target, int count, struct nameandversion *nameandversion, int *remaining) {
 	retvalue result, r;
 	int i;
 
 	result = RET_NOTHING;
 	for (i = 0 ; i < count ; i++){
 		r = target_removepackage(target, distribution->logger,
-				names[i], NULL, trackingdata);
+				nameandversion[i].name, nameandversion[i].version, trackingdata);
 		RET_UPDATE(distribution->status, r);
 		if (RET_IS_OK(r)) {
-			if (!gotremoved[i])
+			if (!nameandversion[i].found)
 				(*remaining)--;
-			gotremoved[i] = true;
+			nameandversion[i].found = true;
 		}
 		RET_UPDATE(result, r);
 	}
@@ -723,8 +723,9 @@ static retvalue remove_from_target(struct distribution *distribution, struct tra
 ACTION_D(y, n, y, remove) {
 	retvalue result, r;
 	struct distribution *distribution;
+	struct nameandversion data[argc-2];
 	struct target *t;
-	bool *gotremoved;
+	char *delimiter;
 	int remaining;
 
 	trackingdb tracks;
@@ -758,12 +759,15 @@ ACTION_D(y, n, y, remove) {
 		}
 	}
 
+	for (int i = 0 ; i < argc-2 ; i++) {
+		data[i].found = false;
+		data[i].name = argv[2 + i];
+		data[i].version = NULL;
+	}
+
 	remaining = argc-2;
-	gotremoved = nzNEW(argc - 2, bool);
 	result = RET_NOTHING;
-	if (FAILEDTOALLOC(gotremoved))
-		result = RET_ERROR_OOM;
-	else for (t = distribution->targets ; t != NULL ; t = t->next) {
+	for (t = distribution->targets ; t != NULL ; t = t->next) {
 		 if (!target_matches(t, components, architectures, packagetypes))
 			 continue;
 		 r = target_initpackagesdb(t, READWRITE);
@@ -774,8 +778,8 @@ ACTION_D(y, n, y, remove) {
 				 (distribution->tracking != dt_NONE)
 				 	? &trackingdata
 					: NULL,
-				 t, argc-2, argv+2,
-				 &remaining, gotremoved);
+				 t, argc-2, data,
+				 &remaining);
 		 RET_UPDATE(result, r);
 		 r = target_closepackagesdb(t);
 		 RET_UPDATE(distribution->status, r);
@@ -796,19 +800,20 @@ ACTION_D(y, n, y, remove) {
 		int i = argc - 2;
 
 		(void)fputs("Not removed as not found: ", stderr);
-		while (i > 0) {
-			i--;
-			assert(gotremoved != NULL);
-			if (!gotremoved[i]) {
-				(void)fputs(argv[2 + i], stderr);
+		delimiter = "";
+		for (i = 0; i < argc - 2; i++) {
+			if (!data[i].found) {
+				if (data[i].version == NULL) {
+					fprintf(stderr, "%s%s", delimiter, data[i].name);
+				} else {
+					fprintf(stderr, "%s%s=%s", delimiter, data[i].name, data[i].version);
+				}
 				remaining--;
-				if (remaining > 0)
-					(void)fputs(", ", stderr);
+				delimiter = ", ";
 			}
 		}
 		(void)fputc('\n', stderr);
 	}
-	free(gotremoved);
 	return result;
 }
 

From 84108ed06408d804972dd0dd351914cdb0b9b1a4 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 23 Feb 2017 14:20:08 +0100
Subject: [PATCH 45/70] table_addrecord: Support non-duplicate tables

The DB_NODUPDATA flag may only be specified if the underlying database
has been configured to support sorted duplicates. Thus do not set the
DB_NODUPDATA flag when the database does not support duplicates. To
avoid querying the flags on each call, save the flags when opening the
table.
---
 database.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/database.c b/database.c
index d0dca04d..fbf3ae90 100644
--- a/database.c
+++ b/database.c
@@ -957,6 +957,7 @@ struct table {
 	DB *berkeleydb;
 	bool *flagreset;
 	bool readonly, verbose;
+	uint32_t flags;
 };
 
 static void table_printerror(struct table *table, int dbret, const char *action) {
@@ -1233,7 +1234,7 @@ retvalue table_addrecord(struct table *table, const char *key, const char *data,
 	SETDBT(Key, key);
 	SETDBTl(Data, data, datalen + 1);
 	dbret = table->berkeleydb->put(table->berkeleydb, NULL,
-			&Key, &Data, DB_NODUPDATA);
+			&Key, &Data, ISSET(table->flags, DB_DUPSORT) ? DB_NODUPDATA : 0);
 	if (dbret != 0 && !(ignoredups && dbret == DB_KEYEXIST)) {
 		table_printerror(table, dbret, "put");
 		return RET_DBERR(dbret);
@@ -1757,6 +1758,7 @@ static retvalue database_table(const char *filename, const char *subtable, enum
 		table->subname = NULL;
 	table->readonly = ISSET(flags, DB_RDONLY);
 	table->verbose = rdb_verbose;
+	table->flags = flags;
 	r = database_opentable(filename, subtable, type, flags,
 			&table->berkeleydb);
 	if (RET_WAS_ERROR(r)) {

From f5887757ade382b3049cedd6ae49db4624b32dd9 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 12:39:44 +0100
Subject: [PATCH 46/70] Support adding the same upstream tarball twice

DB_DUPSORT allows duplicate keys in the database, but not duplicate
key/value pairs. Only if the duplicate data items are unsorted,
applications may store identical duplicate data items.

Since the references.db stores a the filekey mapping to the
codename|component|architecture triplet, there might be identical
duplicates, when upstream tarballs are references by multiple version.

Therefore switch references.db from DB_DUPSORT to DB_DUP.

closes bug #9
---
 database.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/database.c b/database.c
index fbf3ae90..bb9d89ee 100644
--- a/database.c
+++ b/database.c
@@ -243,7 +243,7 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 		free(fullfilename);
 		return RET_DBERR(dbret);
 	}
-	if (type == dbt_BTREEDUP || type == dbt_BTREEPAIRS) {
+	if (type == dbt_BTREEPAIRS) {
 		dbret = table->set_flags(table, DB_DUPSORT);
 		if (dbret != 0) {
 			table->err(table, dbret, "db_set_flags(DB_DUPSORT):");
@@ -251,6 +251,13 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 			free(fullfilename);
 			return RET_DBERR(dbret);
 		}
+	} else if (type == dbt_BTREEDUP) {
+		dbret = table->set_flags(table, DB_DUP);
+		if (dbret != 0) {
+			table->err(table, dbret, "db_set_flags(DB_DUP):");
+			(void)table->close(table, 0);
+			return RET_DBERR(dbret);
+		}
 	}
 	if (type == dbt_BTREEPAIRS) {
 		dbret = table->set_dup_compare(table,  paireddatacompare);

From 71ba94f168f184af5306d700a34aee7bf2e6e6f6 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 30 Aug 2018 13:10:54 +0200
Subject: [PATCH 47/70] Support multiple versions for removesrc command

---
 tracking.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tracking.c b/tracking.c
index 1d3cef5d..473cc815 100644
--- a/tracking.c
+++ b/tracking.c
@@ -1196,8 +1196,8 @@ static retvalue targetremovesourcepackage(trackingdb t, struct trackedpackage *p
 	component_len = strlen(component);
 	arch_len = strlen(architecture);
 	for (i = 0 ; i < pkg->filekeys.count ; i++) {
-		const char *s, *basefilename, *filekey = pkg->filekeys.values[i];
-		char *packagename;
+		const char *s, *s2, *basefilename, *filekey = pkg->filekeys.values[i];
+		char *packagename, *packageversion;
 		struct package package;
 		struct strlist filekeys;
 		bool savedstaletracking;
@@ -1252,9 +1252,14 @@ static retvalue targetremovesourcepackage(trackingdb t, struct trackedpackage *p
 		packagename = strndup(basefilename, s - basefilename);
 		if (FAILEDTOALLOC(packagename))
 			return RET_ERROR_OOM;
-		r = package_get(target, packagename, NULL, &package);
+		s2 = strrchr(s, '.');
+		packageversion = strndup(s + 1, s2 - s - 1);
+		if (FAILEDTOALLOC(packageversion))
+			return RET_ERROR_OOM;
+		r = package_get(target, packagename, packageversion, &package);
 		if (RET_WAS_ERROR(r)) {
 			free(packagename);
+			free(packageversion);
 			return r;
 		}
 		if (r == RET_NOTHING) {
@@ -1262,16 +1267,18 @@ static retvalue targetremovesourcepackage(trackingdb t, struct trackedpackage *p
 			    && verbose >= -1) {
 				fprintf(stderr,
 "Warning: tracking data might be inconsistent:\n"
-"cannot find '%s' in '%s', but '%s' should be there.\n",
-						packagename, target->identifier,
+"cannot find '%s=%s' in '%s', but '%s' should be there.\n",
+						packagename, packageversion, target->identifier,
 						filekey);
 			}
 			free(packagename);
+			free(packageversion);
 			continue;
 		}
 		// TODO: ugly
 		package.pkgname = packagename;
 		packagename = NULL;
+		free(packageversion);
 
 		r = package_getsource(&package);
 		assert (r != RET_NOTHING);

From 035061f36ec3e95915d8c1ba33783a235801659b Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 6 Feb 2017 12:39:44 +0100
Subject: [PATCH 48/70] Add multiple version management

Closes: #570623
Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 copypackages.c  |   6 +-
 database.c      | 104 +++++++++++++++++++++++---
 database.h      |   2 +-
 docs/reprepro.1 |  15 ++--
 main.c          |  24 +++++-
 release.c       |   2 +-
 target.c        | 191 +++++++++++++++++++++++++++++-------------------
 7 files changed, 243 insertions(+), 101 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index 1afc5706..e3997a89 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -90,12 +90,12 @@ static retvalue list_newpackage(struct package_list *list, struct target *target
 		t = *t_p;
 
 	p_p = &t->packages;
-	while (*p_p != NULL && (c = strcmp(packagename, (*p_p)->name)) < 0)
+	while (*p_p != NULL && (c = cascade_strcmp(packagename, (*p_p)->name, packageversion, (*p_p)->version)) < 0)
 		p_p = &(*p_p)->next;
 	if (*p_p != NULL && c == 0) {
 		// TODO: improve this message..., or some context elsewhere
-		fprintf(stderr, "Multiple occurrences of package '%s'!\n",
-				packagename);
+		fprintf(stderr, "Multiple occurrences of package '%s' with version '%s'!\n",
+				packagename, packageversion);
 		return RET_ERROR_EXIST;
 	}
 	package = zNEW(struct selectedpackage);
diff --git a/database.c b/database.c
index bb9d89ee..547fc406 100644
--- a/database.c
+++ b/database.c
@@ -211,13 +211,13 @@ static retvalue database_hasdatabasefile(const char *filename, /*@out@*/bool *ex
 
 enum database_type {
 	dbt_QUERY,
-	dbt_BTREE, dbt_BTREEDUP, dbt_BTREEPAIRS,
+	dbt_BTREE, dbt_BTREEDUP, dbt_BTREEPAIRS, dbt_BTREEVERSIONS,
 	dbt_HASH,
 	dbt_COUNT /* must be last */
 };
 static const uint32_t types[dbt_COUNT] = {
 	DB_UNKNOWN,
-	DB_BTREE, DB_BTREE, DB_BTREE,
+	DB_BTREE, DB_BTREE, DB_BTREE, DB_BTREE,
 	DB_HASH
 };
 
@@ -243,7 +243,7 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 		free(fullfilename);
 		return RET_DBERR(dbret);
 	}
-	if (type == dbt_BTREEPAIRS) {
+	if (type == dbt_BTREEPAIRS || type == dbt_BTREEVERSIONS) {
 		dbret = table->set_flags(table, DB_DUPSORT);
 		if (dbret != 0) {
 			table->err(table, dbret, "db_set_flags(DB_DUPSORT):");
@@ -268,6 +268,15 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 			return RET_DBERR(dbret);
 		}
 	}
+	if (type == dbt_BTREEVERSIONS) {
+		dbret = table->set_dup_compare(table, debianversioncompare);
+		if (dbret != 0) {
+			table->err(table, dbret, "db_set_dup_compare:");
+			(void)table->close(table, 0);
+			free(fullfilename);
+			return RET_DBERR(dbret);
+		}
+	}
 
 #if DB_VERSION_MAJOR == 5 || DB_VERSION_MAJOR == 6
 #define DB_OPEN(database, filename, name, type, flags) \
@@ -962,6 +971,7 @@ struct cursor {
 struct table {
 	char *name, *subname;
 	DB *berkeleydb;
+	DB *sec_berkeleydb;
 	bool *flagreset;
 	bool readonly, verbose;
 	uint32_t flags;
@@ -971,6 +981,9 @@ static void table_printerror(struct table *table, int dbret, const char *action)
 	char *error_msg;
 
 	switch (dbret) {
+	case DB_MALFORMED_KEY:
+		error_msg = "DB_MALFORMED_KEY: Primary key does not contain the separator '|'.";
+		break;
 	case RET_ERROR_OOM:
 		error_msg = "RET_ERROR_OOM: Out of memory.";
 		break;
@@ -1010,6 +1023,15 @@ retvalue table_close(struct table *table) {
 		return RET_NOTHING;
 	if (table->flagreset != NULL)
 		*table->flagreset = false;
+	if (table->sec_berkeleydb != NULL) {
+		dbret = table->sec_berkeleydb->close(table->sec_berkeleydb, 0);
+		if (dbret != 0) {
+			fprintf(stderr, "db_sec_close(%s, %s): %s\n",
+					table->name, table->subname,
+					db_strerror(dbret));
+			result = RET_DBERR(dbret);
+		}
+	}
 	if (table->berkeleydb == NULL) {
 		assert (table->readonly);
 		dbret = 0;
@@ -1027,9 +1049,10 @@ retvalue table_close(struct table *table) {
 	return result;
 }
 
-retvalue table_getrecord(struct table *table, const char *key, char **data_p, size_t *datalen_p) {
+retvalue table_getrecord(struct table *table, bool secondary, const char *key, char **data_p, size_t *datalen_p) {
 	int dbret;
 	DBT Key, Data;
+	DB *db;
 
 	assert (table != NULL);
 	if (table->berkeleydb == NULL) {
@@ -1041,8 +1064,11 @@ retvalue table_getrecord(struct table *table, const char *key, char **data_p, si
 	CLEARDBT(Data);
 	Data.flags = DB_DBT_MALLOC;
 
-	dbret = table->berkeleydb->get(table->berkeleydb, NULL,
-			&Key, &Data, 0);
+	if (secondary)
+		db = table->sec_berkeleydb;
+	else
+		db = table->berkeleydb;
+	dbret = db->get(db, NULL, &Key, &Data, 0);
 	// TODO: find out what error code means out of memory...
 	if (dbret == DB_NOTFOUND)
 		return RET_NOTHING;
@@ -1331,10 +1357,17 @@ retvalue table_replacerecord(struct table *table, const char *key, const char *d
 }
 
 static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **cursor_p) {
+	DB *berkeleydb;
 	struct cursor *cursor;
 	int dbret;
 
-	if (table->berkeleydb == NULL) {
+	if (table->sec_berkeleydb == NULL) {
+		berkeleydb = table->berkeleydb;
+	} else {
+		berkeleydb = table->sec_berkeleydb;
+	}
+
+	if (berkeleydb == NULL) {
 		assert (table->readonly);
 		*cursor_p = NULL;
 		return RET_NOTHING;
@@ -1347,7 +1380,7 @@ static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **c
 	cursor->cursor = NULL;
 	cursor->flags = flags;
 	cursor->r = RET_OK;
-	dbret = table->berkeleydb->cursor(table->berkeleydb, NULL,
+	dbret = berkeleydb->cursor(berkeleydb, NULL,
 			&cursor->cursor, 0);
 	if (dbret != 0) {
 		table_printerror(table, dbret, "cursor");
@@ -1741,7 +1774,8 @@ retvalue database_haspackages(const char *identifier) {
 /****************************************************************************
  * Open the different types of tables with their needed flags:              *
  ****************************************************************************/
-static retvalue database_table(const char *filename, const char *subtable, enum database_type type, uint32_t flags, /*@out@*/struct table **table_p) {
+static retvalue database_table_secondary(const char *filename, const char *subtable, enum database_type type, uint32_t flags,
+                                         const char *secondary_filename, enum database_type secondary_type, /*@out@*/struct table **table_p) {
 	struct table *table;
 	retvalue r;
 
@@ -1787,10 +1821,41 @@ static retvalue database_table(const char *filename, const char *subtable, enum
 		}
 
 	}
+
+	if (secondary_filename != NULL) {
+		r = database_opentable(secondary_filename, subtable, secondary_type, flags,
+				&table->sec_berkeleydb);
+		if (RET_WAS_ERROR(r)) {
+			table->berkeleydb->close(table->berkeleydb, 0);
+			free(table->subname);
+			free(table->name);
+			free(table);
+			return r;
+		}
+		if (r == RET_NOTHING) {
+			if (ISSET(flags, DB_RDONLY)) {
+				/* sometimes we don't want a return here, when? */
+				table->sec_berkeleydb = NULL;
+				r = RET_OK;
+			} else {
+				table->berkeleydb->close(table->berkeleydb, 0);
+				free(table->subname);
+				free(table->name);
+				free(table);
+				return r;
+			}
+
+		}
+	}
+
 	*table_p = table;
 	return r;
 }
 
+static retvalue database_table(const char *filename, const char *subtable, enum database_type type, uint32_t flags, /*@out@*/struct table **table_p) {
+	return database_table_secondary(filename, subtable, type, flags, NULL, 0, table_p);
+}
+
 retvalue database_openreferences(void) {
 	retvalue r;
 
@@ -1925,11 +1990,21 @@ retvalue database_openpackages(const char *identifier, bool readonly, struct tab
 		return RET_ERROR;
 	}
 
-	r = database_table("packages.db", identifier,
-			dbt_BTREE, readonly?DB_RDONLY:DB_CREATE, &table);
+	r = database_table_secondary("packages.db", identifier,
+			dbt_BTREE, readonly?DB_RDONLY:DB_CREATE,
+			"packagenames.db", dbt_BTREEVERSIONS, &table);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
 		return r;
+
+	if (table->berkeleydb != NULL && table->sec_berkeleydb != NULL) {
+		r = table->berkeleydb->associate(table->berkeleydb, NULL,
+				table->sec_berkeleydb, get_package_name, 0);
+		if (RET_WAS_ERROR(r)) {
+			return r;
+		}
+	}
+
 	table->flagreset = &rdb_packagesdatabaseopen;
 	rdb_packagesdatabaseopen = true;
 	*table_p = table;
@@ -1943,7 +2018,12 @@ retvalue database_listpackages(struct strlist *identifiers) {
 
 /* drop a database */
 retvalue database_droppackages(const char *identifier) {
-	return database_dropsubtable("packages.db", identifier);
+	retvalue r;
+
+	r = database_dropsubtable("packages.db", identifier);
+	if (RET_IS_OK(r))
+		r = database_dropsubtable("packagenames.db", identifier);
+	return r;
 }
 
 retvalue database_openfiles(void) {
diff --git a/database.h b/database.h
index 25bc3ca0..439ca037 100644
--- a/database.h
+++ b/database.h
@@ -35,7 +35,7 @@ retvalue database_haspackages(const char *);
 
 bool table_recordexists(struct table *, const char *);
 /* retrieve a record from the database, return RET_NOTHING if there is none: */
-retvalue table_getrecord(struct table *, const char *, /*@out@*/char **, /*@out@*/ /*@null@*/ size_t *);
+retvalue table_getrecord(struct table *, bool, const char *, /*@out@*/char **, /*@out@*/ /*@null@*/ size_t *);
 retvalue table_gettemprecord(struct table *, const char *, /*@out@*//*@null@*/const char **, /*@out@*//*@null@*/size_t *);
 retvalue table_getpair(struct table *, const char *, const char *, /*@out@*/const char **, /*@out@*/size_t *);
 
diff --git a/docs/reprepro.1 b/docs/reprepro.1
index 88e4c045..0b8fa7c6 100644
--- a/docs/reprepro.1
+++ b/docs/reprepro.1
@@ -664,11 +664,12 @@ List the versions of the specified package in all distributions.
 .B lsbycomponent \fIpackage-name\fP
 Like ls, but group by component (and print component names).
 .TP
-.B remove \fIcodename\fP \fIpackage-names\fP
-Delete all packages in the specified distribution,
+.B remove \fIcodename\fP \fIpackage-names\fP\fR[\fP=\fIversion\fP\fR]\fP \fI...\fP
+Delete packages in the specified distribution,
 that have package name listed as argument.
-(i.e. remove all packages \fBlist\fP with the same arguments and options
-would list, except that an empty package list is not allowed.)
+Package versions must be specified by appending '\fB=\fP' and the
+version to the name (without spaces). When no version is specified, the latest
+package version is removed.
 
 Note that like any other operation removing or replacing a package,
 the old package's files are unreferenced and thus may be automatically
@@ -884,11 +885,13 @@ used files.
 Check all source package tracking information for the given distributions
 for files no longer to keep.
 .TP
-.B copy \fIdestination-codename\fP \fIsource-codename\fP \fIpackages...\fP
+.B copy \fIdestination-codename\fP \fIsource-codename\fP \fIpackage\fP\fR[\fP=\fIversion\fP\fR]\fP \fI...\fP
 Copy the given packages from one distribution to another.
 The packages are copied verbatim, no override files are consulted.
 Only components and architectures present in the source distribution are
-copied.
+copied. Package versions must be specified by appending '\fB=\fP' and the
+version to the name (without spaces). When no version is specified, the latest
+package version is copied.
 .TP
 .B copysrc \fIdestination-codename\fP \fIsource-codename\fP \fIsource-package\fP \fR[\fP\fIversions\fP\fR]\fP
 look at each package
diff --git a/main.c b/main.c
index ae55caa0..d2cf1463 100644
--- a/main.c
+++ b/main.c
@@ -761,8 +761,13 @@ ACTION_D(y, n, y, remove) {
 
 	for (int i = 0 ; i < argc-2 ; i++) {
 		data[i].found = false;
-		data[i].name = argv[2 + i];
-		data[i].version = NULL;
+		r = splitnameandversion(argv[2 + i], &data[i].name, &data[i].version);
+		if (RET_WAS_ERROR(r)) {
+			for (i-- ; i >= 0 ; i--) {
+				splitnameandversion_done(&data[i].name, &data[i].version);
+			}
+			return r;
+		}
 	}
 
 	remaining = argc-2;
@@ -814,6 +819,9 @@ ACTION_D(y, n, y, remove) {
 		}
 		(void)fputc('\n', stderr);
 	}
+	for (int i = 0; i < argc - 2; i++) {
+		splitnameandversion_done(&data[i].name, &data[i].version);
+	}
 	return result;
 }
 
@@ -1986,14 +1994,22 @@ ACTION_D(y, n, y, copy) {
 		return result;
 
 	for (i = 0; i < argc-3; i++) {
-		data[i].name = argv[3 + i];
-		data[i].version = NULL;
+		result = splitnameandversion(argv[3 + i], &data[i].name, &data[i].version);
+		if (RET_WAS_ERROR(result)) {
+			for (i-- ; i >= 0 ; i--) {
+				splitnameandversion_done(&data[i].name, &data[i].version);
+			}
+			return result;
+		}
 	}
 	data[i].name = NULL;
 	data[i].version = NULL;
 
 	result = copy_by_name(destination, source, data,
 			components, architectures, packagetypes);
+	for (i = 0; i < argc - 3; i++) {
+		splitnameandversion_done(&data[i].name, &data[i].version);
+	}
 	return result;
 }
 
diff --git a/release.c b/release.c
index 1b2ad516..8dd953c4 100644
--- a/release.c
+++ b/release.c
@@ -441,7 +441,7 @@ static retvalue release_usecached(struct release *release,
 
 		if (filename[ic] == NULL)
 			continue;
-		r = table_getrecord(release->cachedb, filename[ic],
+		r = table_getrecord(release->cachedb, false, filename[ic],
 				&combinedchecksum, NULL);
 		if (!RET_IS_OK(r)) {
 			result = r;
diff --git a/target.c b/target.c
index 584bac23..6645b573 100644
--- a/target.c
+++ b/target.c
@@ -227,12 +227,11 @@ retvalue target_closepackagesdb(struct target *target) {
 retvalue package_remove(struct package *old, struct logger *logger, struct trackingdata *trackingdata) {
 	struct strlist files;
 	retvalue result, r;
+	char *key;
 
 	assert (old->target != NULL && old->target->packages != NULL);
 
-	if (logger != NULL) {
-		(void)package_getversion(old);
-	}
+	(void)package_getversion(old);
 	r = old->target->getfilekeys(old->control, &files);
 	if (RET_WAS_ERROR(r)) {
 		return r;
@@ -243,7 +242,9 @@ retvalue package_remove(struct package *old, struct logger *logger, struct track
 	if (verbose > 0)
 		printf("removing '%s=%s' from '%s'...\n",
 				old->name, old->version, old->target->identifier);
-	result = table_deleterecord(old->target->packages, old->name, false);
+	key = package_primarykey(old->name, old->version);
+	result = table_deleterecord(old->target->packages, key, false);
+	free(key);
 	if (RET_IS_OK(result)) {
 		old->target->wasmodified = true;
 		if (trackingdata != NULL && old->source != NULL
@@ -319,7 +320,7 @@ retvalue package_remove_by_cursor(struct package_cursor *tc, struct logger *logg
 	if (verbose > 0)
 		printf("removing '%s=%s' from '%s'...\n",
 				old->name, old->version, old->target->identifier);
-	result = cursor_delete(target->packages, tc->cursor, old->name, NULL);
+	result = cursor_delete(target->packages, tc->cursor, old->name, old->version);
 	if (RET_IS_OK(result)) {
 		old->target->wasmodified = true;
 		if (trackingdata != NULL && old->source != NULL
@@ -348,6 +349,7 @@ retvalue package_remove_by_cursor(struct package_cursor *tc, struct logger *logg
 static retvalue addpackages(struct target *target, const char *packagename, const char *controlchunk, const char *version, const struct strlist *files, /*@null@*/const struct package *old, /*@null@*/const struct strlist *oldfiles, /*@null@*/struct logger *logger, /*@null@*/struct trackingdata *trackingdata, architecture_t architecture, /*@null@*/const char *causingrule, /*@null@*/const char *suitefrom) {
 
 	retvalue result = RET_OK, r;
+	char *key;
 	struct table *table = target->packages;
 	enum filetype filetype;
 
@@ -369,13 +371,19 @@ static retvalue addpackages(struct target *target, const char *packagename, cons
 
 	/* Add package to the distribution's database */
 
-	if (old != NULL) {
-		r = table_replacerecord(table, packagename, controlchunk);
-
-	} else {
-		r = table_adduniqrecord(table, packagename, controlchunk);
+	if (old != NULL && old->control != NULL) {
+		key = package_primarykey(old->name, old->version);
+		r = table_deleterecord(table, key, false);
+		free(key);
+		if (RET_WAS_ERROR(r)) {
+			return r;
+		}
 	}
 
+	key = package_primarykey(packagename, version);
+	r = table_adduniqrecord(table, key, controlchunk);
+	free(key);
+
 	if (RET_WAS_ERROR(r))
 		return r;
 
@@ -411,67 +419,87 @@ static retvalue addpackages(struct target *target, const char *packagename, cons
 }
 
 retvalue target_addpackage(struct target *target, struct logger *logger, const char *name, const char *version, const char *control, const struct strlist *filekeys, bool downgrade, struct trackingdata *trackingdata, architecture_t architecture, const char *causingrule, const char *suitefrom) {
-	struct strlist oldfilekeys, *ofk;
+	struct strlist oldfilekeys, *ofk = NULL;
 	char *newcontrol;
-	struct package old, *old_p;
+	struct package_cursor iterator = {NULL};
+	struct package old;
 	retvalue r;
 
 	assert(target->packages!=NULL);
 
-	r = package_get(target, name, NULL, &old);
-	if (RET_WAS_ERROR(r))
+	r = package_get(target, name, version, &old);
+	if (RET_WAS_ERROR(r)) {
+		package_done(&old);
 		return r;
-	if (r == RET_NOTHING) {
-		old_p = NULL;
-		ofk = NULL;
-		setzero(struct package, &old);
-	} else {
-		old_p = &old;
-		r = package_getversion(&old);
-		if (RET_WAS_ERROR(r) && !IGNORING(brokenold,
-"Error parsing old version!\n")) {
+	} else if (RET_IS_OK(r)) {
+		if (!downgrade) {
+			fprintf(stderr, "Skipping inclusion of '%s' '%s' in '%s', as this version already exists.\n",
+					name, version, target->identifier);
 			package_done(&old);
+			return RET_NOTHING;
+		} else {
+			r = package_getversion(&old);
+			if (RET_WAS_ERROR(r) && !IGNORING(brokenold, "Error parsing old version!\n")) {
+				package_done(&old);
+				return r;
+			}
+			fprintf(stderr, "Warning: replacing '%s' version '%s' with equal version '%s' in '%s'!\n",
+			        name, old.version, version, target->identifier);
+		}
+	} else if (true) {
+		package_done(&old);
+		r = package_openduplicateiterator(target, name, 0, &iterator);
+		if (RET_WAS_ERROR(r)) {
 			return r;
 		}
 		if (RET_IS_OK(r)) {
-			int versioncmp;
-
-			r = dpkgversions_cmp(version, old.version,
-					&versioncmp);
-			if (RET_WAS_ERROR(r)) {
-				if (!IGNORING(brokenversioncmp,
-"Parse errors processing versions of %s.\n", name)) {
-					package_done(&old);
-					return r;
-				}
-			} else {
-				if (versioncmp <= 0) {
-					/* new Version is not newer than
-					 * old version */
+			r = package_getversion(&iterator.current);
+			if (RET_WAS_ERROR(r) && !IGNORING(brokenold, "Error parsing old version!\n")) {
+				retvalue r2 = package_closeiterator(&iterator);
+				RET_ENDUPDATE(r, r2);
+				return r;
+			}
+			if (RET_IS_OK(r)) {
+				int versioncmp;
+
+				r = dpkgversions_cmp(version, iterator.current.version, &versioncmp);
+				if (RET_WAS_ERROR(r)) {
+					if (!IGNORING(brokenversioncmp, "Parse errors processing versions of %s.\n", name)) {
+						retvalue r2 = package_closeiterator(&iterator);
+						RET_ENDUPDATE(r, r2);
+						return r;
+					}
+				} else if (versioncmp < 0) {
+					// new Version is older than the old version that will be replaced
 					if (!downgrade) {
 						fprintf(stderr,
 "Skipping inclusion of '%s' '%s' in '%s', as it has already '%s'.\n",
 							name, version,
 							target->identifier,
-							old.version);
+							iterator.current.version);
 						package_done(&old);
 						return RET_NOTHING;
-					} else if (versioncmp < 0) {
-						fprintf(stderr,
-"Warning: downgrading '%s' from '%s' to '%s' in '%s'!\n", name,
-							old.version,
-							version,
-							target->identifier);
 					} else {
 						fprintf(stderr,
-"Warning: replacing '%s' version '%s' with equal version '%s' in '%s'!\n", name,
-							old.version,
+"Warning: downgrading '%s' from '%s' to '%s' in '%s'!\n", name,
+							iterator.current.version,
 							version,
 							target->identifier);
 					}
 				}
+				old.target = target;
+				old.name = iterator.current.name;
+				old.control = iterator.current.control;
+				old.controllen = iterator.current.controllen;
+				old.version = iterator.current.version;
 			}
 		}
+	} else {
+		// Keep all package versions in the archive.
+		package_done(&old);
+	}
+
+	if (old.name != NULL) {
 		r = target->getfilekeys(old.control, &oldfilekeys);
 		ofk = &oldfilekeys;
 		if (RET_WAS_ERROR(r)) {
@@ -480,6 +508,10 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 				ofk = NULL;
 			} else {
 				package_done(&old);
+				if (iterator.cursor != NULL) {
+					retvalue r2 = package_closeiterator(&iterator);
+					RET_ENDUPDATE(r, r2);
+				}
 				return r;
 			}
 		} else if (trackingdata != NULL) {
@@ -492,12 +524,16 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 					ofk = NULL;
 				} else {
 					package_done(&old);
+					if (iterator.cursor != NULL) {
+						retvalue r2 = package_closeiterator(&iterator);
+						RET_ENDUPDATE(r, r2);
+					}
 					return r;
 				}
 			}
 		}
-
 	}
+
 	newcontrol = NULL;
 	r = description_addpackage(target, name, control, &newcontrol);
 	if (RET_IS_OK(r))
@@ -506,7 +542,7 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 		r = addpackages(target, name, control,
 			version,
 			filekeys,
-			old_p, ofk,
+			&old, ofk,
 			logger,
 			trackingdata, architecture,
 			causingrule, suitefrom);
@@ -519,6 +555,12 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 	}
 	free(newcontrol);
 	package_done(&old);
+
+	if (iterator.cursor != NULL) {
+		retvalue r2;
+		r2 = package_closeiterator(&iterator);
+		RET_ENDUPDATE(r, r2);
+	}
 	return r;
 }
 
@@ -555,35 +597,30 @@ retvalue target_checkaddpackage(struct target *target, const char *name, const c
 			package_done(&old);
 			return r;
 		}
-		if (versioncmp <= 0) {
-			r = RET_NOTHING;
-			if (versioncmp < 0) {
-				if (!permitnewerold) {
-					fprintf(stderr,
+		if (versioncmp < 0) {
+			if (!permitnewerold) {
+				fprintf(stderr,
 "Error: trying to put version '%s' of '%s' in '%s',\n"
 "while there already is the stricly newer '%s' in there.\n"
 "(To ignore this error add Permit: older_version.)\n",
-						version, name,
-						target->identifier,
-						old.version);
-					r = RET_ERROR;
-				} else if (verbose >= 0) {
-					printf(
-"Warning: trying to put version '%s' of '%s' in '%s',\n"
-"while there already is '%s' in there.\n",
-						version, name,
-						target->identifier,
-						old.version);
-				}
+					version, name,
+					target->identifier,
+					old.version);
+				package_done(&old);
+				return RET_ERROR;
 			} else if (verbose > 2) {
-					printf(
+				printf("Puting version '%s' of '%s' in '%s', while there already is '%s' in there.\n",
+					version, name, target->identifier, old.version);
+			}
+		} else if (versioncmp == 0) {
+			if (verbose > 2) {
+				printf(
 "Will not put '%s' in '%s', as already there with same version '%s'.\n",
-						name, target->identifier,
-						old.version);
-
+					name, target->identifier,
+					old.version);
 			}
 			package_done(&old);
-			return r;
+			return RET_NOTHING;
 		}
 		r = target->getfilekeys(old.control, &oldfilekeys);
 		ofk = &oldfilekeys;
@@ -911,8 +948,6 @@ retvalue package_get(struct target *target, const char *name, const char *versio
 	retvalue result, r;
 	bool database_closed;
 
-	assert (version == NULL); /* not yet implemented */
-
 	memset(pkg, 0, sizeof(*pkg));
 
 	database_closed = target->packages == NULL;
@@ -922,8 +957,16 @@ retvalue package_get(struct target *target, const char *name, const char *versio
 		if (RET_WAS_ERROR(r))
 			return r;
 	}
-	result = table_getrecord(target->packages, name,
-			&pkg->pkgchunk, &pkg->controllen);
+
+	if (version == NULL) {
+		result = table_getrecord(target->packages, true, name,
+				&pkg->pkgchunk, &pkg->controllen);
+	} else {
+		char *key = package_primarykey(name, version);
+		result = table_getrecord(target->packages, false, key,
+				&pkg->pkgchunk, &pkg->controllen);
+		free(key);
+	}
 	if (RET_IS_OK(result)) {
 		pkg->target = target;
 		pkg->name = name;

From 14c585099ee52a85c6e7e2760e20d387556626e1 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 28 Feb 2017 18:13:40 +0100
Subject: [PATCH 49/70] Add Limit option

Limit the number of versions of a package per distribution,
architecture, component, and type. The limit must be a number. If the
number is positive, all old package version that exceed these limit will
be removed when a new package version is added. If the number is zero or
negative, all package version will be kept. By default only one package
version will be kept.
---
 configparser.h  |  5 +++++
 distribution.c  |  3 +++
 distribution.h  |  1 +
 docs/reprepro.1 |  7 +++++++
 target.c        | 17 +++++++++++++++--
 5 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/configparser.h b/configparser.h
index 7442b977..e583b49f 100644
--- a/configparser.h
+++ b/configparser.h
@@ -226,6 +226,11 @@ static retvalue configparser_ ## sname ## _set_ ## name(UNUSED(void *dummy), con
 	struct sname *item = data; \
 	return config_gettruth(iter, name, &item->field); \
 }
+#define CFnumberSETPROC(sname, minval, maxval, field) \
+static retvalue configparser_ ## sname ## _set_ ## field(UNUSED(void *dummy), UNUSED(const char *name), void *data, struct configiterator *iter) { \
+	struct sname *item = data; \
+	return config_getnumber(iter, name, &item->field, minval, maxval); \
+}
 #define CFallSETPROC(sname, field) \
 static retvalue configparser_ ## sname ## _set_ ## field(UNUSED(void *dummy), UNUSED(const char *name), void *data, struct configiterator *iter) { \
 	struct sname *item = data; \
diff --git a/distribution.c b/distribution.c
index f7144d61..d56a8dd8 100644
--- a/distribution.c
+++ b/distribution.c
@@ -223,6 +223,7 @@ CFstartparse(distribution) {
 	if (FAILEDTOALLOC(n))
 		return RET_ERROR_OOM;
 	/* set some default value: */
+	n->limit = 1;
 	r = exportmode_init(&n->udeb, true, NULL, "Packages");
 	if (RET_WAS_ERROR(r)) {
 		(void)distribution_free(n);
@@ -401,6 +402,7 @@ CFallSETPROC(distribution, label)
 CFallSETPROC(distribution, description)
 CFallSETPROC(distribution, signed_by)
 CFsignwithSETPROC(distribution, signwith)
+CFnumberSETPROC(distribution, -1, LLONG_MAX, limit)
 CFfileSETPROC(distribution, deb_override)
 CFfileSETPROC(distribution, udeb_override)
 CFfileSETPROC(distribution, dsc_override)
@@ -471,6 +473,7 @@ static const struct configfield distributionconfigfields[] = {
 	CF("DscOverride",	distribution,	dsc_override),
 	CF("FakeComponentPrefix", distribution,	fakecomponentprefix),
 	CF("Label",		distribution,	label),
+	CF("Limit",		distribution,   limit),
 	CF("Log",		distribution,	logger),
 	CF("NotAutomatic",	distribution,	notautomatic),
 	CF("ButAutomaticUpgrades", distribution, butautomaticupgrades),
diff --git a/distribution.h b/distribution.h
index 31105241..bc658b5a 100644
--- a/distribution.h
+++ b/distribution.h
@@ -52,6 +52,7 @@ struct distribution {
 	struct strlist pulls;
 	/* the key to sign with, may have no entries to mean unsigned: */
 	struct strlist signwith;
+	long long limit;
 	/* the override file to use by default */
 	/*@null@*/char *deb_override, *udeb_override, *dsc_override;
 	/* fake component prefix (and codename antisuffix) for Release files: */
diff --git a/docs/reprepro.1 b/docs/reprepro.1
index 0b8fa7c6..a9d3aec3 100644
--- a/docs/reprepro.1
+++ b/docs/reprepro.1
@@ -1566,6 +1566,13 @@ until no file is needed any more.
 .B needsources
 Not yet implemented.
 .TP
+.B Limit
+Limit the number of versions of a package per distribution, architecture,
+component, and type. The limit must be a number. If the number is positive,
+all old package version that exceed these limit will be removed when a new
+package version is added. If the number is zero or negative, all package
+version will be kept. By default only one package version will be kept.
+.TP
 .B Log
 Specify a file to log additions and removals of this distribution
 into and/or external scripts to call when something is added or
diff --git a/target.c b/target.c
index 6645b573..67d64787 100644
--- a/target.c
+++ b/target.c
@@ -446,9 +446,9 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 			fprintf(stderr, "Warning: replacing '%s' version '%s' with equal version '%s' in '%s'!\n",
 			        name, old.version, version, target->identifier);
 		}
-	} else if (true) {
+	} else if (target->distribution->limit > 0) {
 		package_done(&old);
-		r = package_openduplicateiterator(target, name, 0, &iterator);
+		r = package_openduplicateiterator(target, name, target->distribution->limit - 1, &iterator);
 		if (RET_WAS_ERROR(r)) {
 			return r;
 		}
@@ -557,7 +557,20 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 	package_done(&old);
 
 	if (iterator.cursor != NULL) {
+		// Remove all older versions (that exceed the current limit)
 		retvalue r2;
+		while(package_next(&iterator)) {
+			r2 = package_getversion(&iterator.current);
+			RET_UPDATE(r, r2);
+			if (RET_WAS_ERROR(r2))
+				continue;
+			if (strcmp(version, iterator.current.version) == 0) {
+				// Do not remove the newly added package!
+				continue;
+			}
+			r2 = package_remove_by_cursor(&iterator, logger, trackingdata);
+			RET_UPDATE(r, r2);
+		}
 		r2 = package_closeiterator(&iterator);
 		RET_ENDUPDATE(r, r2);
 	}

From 9b43ffdeb5730df1538d492043de563c0c73b0f7 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 7 Feb 2017 17:20:29 +0100
Subject: [PATCH 50/70] [testsuite] Add multiversion test cases

---
 .gitignore                        |   1 +
 tests/Makefile.am                 |   2 +
 tests/basic.sh                    |  45 +++++
 tests/multiversion.sh             | 266 ++++++++++++++++++++++++++++++
 tests/shunit2-helper-functions.sh |   1 +
 5 files changed, 315 insertions(+)
 create mode 100755 tests/multiversion.sh

diff --git a/.gitignore b/.gitignore
index 20d2586a..4e41cbb5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,4 @@ Makefile
 /tests/.deps/
 /tests/testpkgs/
 /tests/testrepo/
+/tests/upstreamrepo/
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 115a567b..709e79f0 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -50,11 +50,13 @@ revoked.pkey \
 withsubkeys.key \
 withsubkeys-works.key \
 basic.sh \
+multiversion.sh \
 shunit2-helper-functions.sh
 MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
 
 check:
 	./basic.sh
+	./multiversion.sh
 
 clean-local:
 	rm -rf testrepo testpkgs
diff --git a/tests/basic.sh b/tests/basic.sh
index f2845be4..8e1f4a47 100755
--- a/tests/basic.sh
+++ b/tests/basic.sh
@@ -94,6 +94,51 @@ test_limit() {
 	assertEquals "buster|main|$ARCH: hello 2.9-2" "$($REPREPRO -b $REPO list buster)"
 }
 
+test_older_version() {
+	cat >> $REPO/conf/incoming <<EOF
+Name: buster-upload
+IncomingDir: incoming
+TempDir: tmp
+Allow: buster
+Permit: older_version
+EOF
+	echo "Limit: 3" >> $REPO/conf/distributions
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	mkdir -p "$REPO/incoming"
+	cp "$PKGS/hello_2.9-2_${ARCH}.changes" "$PKGS/hello-addons_2.9-2_all.deb" "$PKGS/hello_2.9-2_${ARCH}.deb" "$PKGS/hello_2.9-2.dsc" "$PKGS/hello_2.9.orig.tar.gz" "$PKGS/hello_2.9-2.debian.tar.xz" "$REPO/incoming"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO processincoming buster-upload hello_2.9-2_${ARCH}.changes
+	assertEquals "hello | 2.9-2 | buster | $ARCH, source" "$($REPREPRO -b $REPO ls hello)"
+	cp "$PKGS/hello_2.9-1_${ARCH}.changes" "$PKGS/hello-addons_2.9-1_all.deb" "$PKGS/hello_2.9-1_${ARCH}.deb" "$PKGS/hello_2.9-1.dsc" "$PKGS/hello_2.9.orig.tar.gz" "$PKGS/hello_2.9-1.debian.tar.xz" "$REPO/incoming"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO processincoming buster-upload hello_2.9-1_${ARCH}.changes
+	assertEquals "\
+hello | 2.9-2 | buster | $ARCH, source
+hello | 2.9-1 | buster | $ARCH, source" "$($REPREPRO -b $REPO ls hello)"
+}
+
+test_too_old_version() {
+	# Allow only one version per package in the archive
+	# Test if uploading an older version will not replace the newer version
+	# in the archive.
+	cat >> $REPO/conf/incoming <<EOF
+Name: buster-upload
+IncomingDir: incoming
+TempDir: tmp
+Allow: buster
+Permit: older_version
+EOF
+	echo "Limit: 1" >> $REPO/conf/distributions
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	mkdir -p "$REPO/incoming"
+	cp "$PKGS/hello_2.9-2_${ARCH}.changes" "$PKGS/hello-addons_2.9-2_all.deb" "$PKGS/hello_2.9-2_${ARCH}.deb" "$PKGS/hello_2.9-2.dsc" "$PKGS/hello_2.9.orig.tar.gz" "$PKGS/hello_2.9-2.debian.tar.xz" "$REPO/incoming"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO processincoming buster-upload hello_2.9-2_${ARCH}.changes
+	assertEquals "hello | 2.9-2 | buster | $ARCH, source" "$($REPREPRO -b $REPO ls hello)"
+	cp "$PKGS/hello_2.9-1_${ARCH}.changes" "$PKGS/hello-addons_2.9-1_all.deb" "$PKGS/hello_2.9-1_${ARCH}.deb" "$PKGS/hello_2.9-1.dsc" "$PKGS/hello_2.9.orig.tar.gz" "$PKGS/hello_2.9-1.debian.tar.xz" "$REPO/incoming"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO processincoming buster-upload hello_2.9-1_${ARCH}.changes
+	assertEquals "hello | 2.9-2 | buster | $ARCH, source" "$($REPREPRO -b $REPO ls hello)"
+}
+
 test_remove() {
 	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
 	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
diff --git a/tests/multiversion.sh b/tests/multiversion.sh
new file mode 100755
index 00000000..59a995bb
--- /dev/null
+++ b/tests/multiversion.sh
@@ -0,0 +1,266 @@
+#!/bin/sh
+set -u
+
+# Copyright (C) 2017, Benjamin Drung <benjamin.drung@profitbricks.com>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+. "${0%/*}/shunit2-helper-functions.sh"
+
+oneTimeSetUp() {
+	for revision in 1 2 2+deb8u1 10; do
+		mkdir -p "$PKGS"
+		(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster EPOCH="" VERSION=2.9 REVISION=-$revision ../genpackage.sh)
+	done
+}
+
+setUp() {
+	create_repo
+	echo "Limit: -1" >> $REPO/conf/distributions
+}
+
+tearDown() {
+	check_db
+}
+
+four_hellos() {
+	for revision in 1 2 2+deb8u1 10; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb
+	done
+}
+
+test_ls() {
+	(cd $PKGS && PACKAGE=kvm SECTION=main DISTRI=buster VERSION=1.2.1 REVISION=-8 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=kvm SECTION=main DISTRI=buster VERSION=1.2.1 REVISION=-9 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=appdirs SECTION=main DISTRI=buster VERSION=1.3.0 REVISION=-1 ../genpackage.sh)
+	for package in hello_2.9-1 kvm_1.2.1-8 kvm_1.2.1-9 appdirs_1.3.0-1; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/${package}_${ARCH}.deb
+	done
+	assertEquals "\
+kvm | 1.2.1-9 | buster | $ARCH
+kvm | 1.2.1-8 | buster | $ARCH" "$($REPREPRO -b $REPO ls kvm)"
+	assertEquals "\
+buster|main|$ARCH: kvm 1.2.1-9
+buster|main|$ARCH: kvm 1.2.1-8" "$($REPREPRO -b $REPO list buster kvm)"
+}
+
+test_sorting() {
+	four_hellos
+	assertEquals "\
+buster|main|$ARCH: hello 2.9-10
+buster|main|$ARCH: hello 2.9-2+deb8u1
+buster|main|$ARCH: hello 2.9-2
+buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	assertEquals "\
+hello |       2.9-10 | buster | $ARCH
+hello | 2.9-2+deb8u1 | buster | $ARCH
+hello |        2.9-2 | buster | $ARCH
+hello |        2.9-1 | buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+}
+
+test_include_twice() {
+	for revision in 1 2; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb
+	done
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-1_${ARCH}.deb
+	assertEquals "\
+hello | 2.9-2 | buster | $ARCH
+hello | 2.9-1 | buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+}
+
+test_copy_latest() {
+	four_hellos
+	add_distro bullseye "Limit: -1"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster hello hello
+	assertEquals "bullseye|main|$ARCH: hello 2.9-10" "$($REPREPRO -b $REPO list bullseye)"
+}
+
+test_copy_specific() {
+	four_hellos
+	add_distro bullseye "Limit: -1"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster hello=2.9-10 hello=2.9-1 hello=2.9-10
+	assertEquals "\
+bullseye|main|$ARCH: hello 2.9-10
+bullseye|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list bullseye)"
+}
+
+test_remove_latest() {
+	four_hellos
+	add_distro bullseye "Limit: -1"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster hello=2.9-10 hello=2.9-1 hello=2.9-10
+	call $REPREPRO $VERBOSE_ARGS -b $REPO remove bullseye hello
+	assertEquals "\
+hello |       2.9-10 |   buster | $ARCH
+hello | 2.9-2+deb8u1 |   buster | $ARCH
+hello |        2.9-2 |   buster | $ARCH
+hello |        2.9-1 |   buster | $ARCH
+hello |        2.9-1 | bullseye | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+}
+
+test_remove_specific() {
+	four_hellos
+	call $REPREPRO $VERBOSE_ARGS -b $REPO remove buster hello=2.9-2+deb8u1 hellox hello=2.9-2+deb8u1
+	assertEquals "\
+buster|main|$ARCH: hello 2.9-10
+buster|main|$ARCH: hello 2.9-2
+buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_removefilter() {
+	(cd $PKGS && PACKAGE=kvm SECTION=main DISTRI=buster VERSION=1.2.1 REVISION=-8 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=kvm SECTION=main DISTRI=buster VERSION=1.2.1 REVISION=-9 ../genpackage.sh)
+	for package in hello_2.9-1 kvm_1.2.1-8 kvm_1.2.1-9; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/${package}_${ARCH}.deb
+	done
+	assertEquals "\
+buster|main|$ARCH: hello 2.9-1
+buster|main|$ARCH: kvm 1.2.1-9
+buster|main|$ARCH: kvm 1.2.1-8" "$($REPREPRO -b $REPO list buster)"
+
+	add_distro bullseye "Limit: -1"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copy bullseye buster kvm
+	assertEquals "bullseye|main|$ARCH: kvm 1.2.1-9" "$($REPREPRO -b $REPO list bullseye)"
+
+	call $REPREPRO $VERBOSE_ARGS -b $REPO removefilter buster "Package (= kvm)"
+	assertEquals "buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	assertTrue "kvm_1.2.1-8_$ARCH.deb is still in the pool!" "test ! -e $REPO/pool/main/k/kvm/kvm_1.2.1-8_$ARCH.deb"
+	assertTrue "kvm_1.2.1-9_$ARCH.deb is missing from the pool!" "test -e $REPO/pool/main/k/kvm/kvm_1.2.1-9_$ARCH.deb"
+
+	call $REPREPRO $VERBOSE_ARGS -b $REPO removefilter bullseye "Package (= kvm)"
+	assertEquals "buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	assertTrue "kvm_1.2.1-9_$ARCH.deb is still in the pool!" "test ! -e $REPO/pool/main/k/kvm/kvm_1.2.1-9_$ARCH.deb"
+}
+
+test_readd_distribution() {
+	# Test case for https://github.com/profitbricks/reprepro/issues/1
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-1_${ARCH}.deb
+
+	# Add distribution
+	cp $REPO/conf/distributions $REPO/conf/distributions.backup
+	add_distro bullseye "Limit: -1"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb bullseye $PKGS/hello_2.9-2_${ARCH}.deb
+
+	# Remove distribution
+	mv $REPO/conf/distributions.backup $REPO/conf/distributions
+	call $REPREPRO $VERBOSE_ARGS -b $REPO --delete clearvanished
+
+	# Re-add distribution again
+	echo "I: Re-adding bullseye..."
+	add_distro bullseye "Limit: -1"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb bullseye $PKGS/hello_2.9-10_${ARCH}.deb
+	assertEquals "bullseye|main|$ARCH: hello 2.9-10" "$($REPREPRO -b $REPO list bullseye)"
+}
+
+test_limit3() {
+	sed -i 's/^Limit: .*$/Limit: 3/' $REPO/conf/distributions
+	for revision in 1 2 2+deb8u1; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb
+	done
+	assertEquals "\
+buster|main|${ARCH}: hello 2.9-2+deb8u1
+buster|main|${ARCH}: hello 2.9-2
+buster|main|${ARCH}: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-10_${ARCH}.deb
+	assertEquals "\
+buster|main|${ARCH}: hello 2.9-10
+buster|main|${ARCH}: hello 2.9-2+deb8u1
+buster|main|${ARCH}: hello 2.9-2" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_reduce_limit() {
+	for revision in 1 2 2+deb8u1; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb
+	done
+	assertEquals "\
+buster|main|${ARCH}: hello 2.9-2+deb8u1
+buster|main|${ARCH}: hello 2.9-2
+buster|main|${ARCH}: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	sed -i 's/^Limit: .*$/Limit: 1/' $REPO/conf/distributions
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-10_${ARCH}.deb
+	assertEquals "buster|main|${ARCH}: hello 2.9-10" "$($REPREPRO -b $REPO list buster)"
+	assertEquals "\
+Distribution: buster
+Source: hello
+Version: 2.9-10
+Files:
+ pool/main/h/hello/hello_2.9-10_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+}
+
+test_limit_old() {
+	for revision in 1 2 10; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb
+	done
+	assertEquals "\
+buster|main|${ARCH}: hello 2.9-10
+buster|main|${ARCH}: hello 2.9-2
+buster|main|${ARCH}: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	sed -i 's/^Limit: .*$/Limit: 2/' $REPO/conf/distributions
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-2+deb8u1_${ARCH}.deb
+	assertEquals "\
+buster|main|${ARCH}: hello 2.9-10
+buster|main|${ARCH}: hello 2.9-2+deb8u1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_update_packages() {
+	# Test case for https://github.com/profitbricks/reprepro/issues/6
+	local upstream_repo
+	upstream_repo="${0%/*}/upstreamrepo"
+
+	four_hellos
+	rm -rf "$upstream_repo"
+	mv "$REPO" "$upstream_repo"
+
+	mkdir -p "$REPO/conf"
+	cat > "$REPO/conf/distributions" <<EOF
+Origin: Icinga2
+Label: Icinga2
+Suite: icinga-stretch
+Codename: icinga-stretch
+Description: Icinga2 packages for Debian Stretch
+Architectures: $ARCH
+Components: main
+Update: icinga-stretch
+Log: icinga2.log
+Limit: -1
+EOF
+	cat > "$REPO/conf/updates" <<EOF
+Name: icinga-stretch
+Method: file://$(readlink -f $upstream_repo)
+Suite: buster
+Components: main
+Architectures: $ARCH
+VerifyRelease: blindtrust
+GetInRelease: no
+EOF
+	call $REPREPRO $VERBOSE_ARGS -b $REPO --noskipold update
+	assertEquals "icinga-stretch|main|$ARCH: hello 2.9-10" "$($REPREPRO -b $REPO list icinga-stretch)"
+
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb icinga-stretch $PKGS/hello_2.9-2_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO --noskipold update
+	assertEquals "\
+icinga-stretch|main|$ARCH: hello 2.9-10
+icinga-stretch|main|$ARCH: hello 2.9-2" "$($REPREPRO -b $REPO list icinga-stretch)"
+}
+
+test_includedsc_sources() {
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedsc buster $PKGS/hello_2.9-1.dsc
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedsc buster $PKGS/hello_2.9-2.dsc
+	assertEquals "\
+buster|main|source: hello 2.9-2
+buster|main|source: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+
+	call $REPREPRO $VERBOSE_ARGS -b $REPO removesrc buster hello 2.9-1
+	assertEquals "buster|main|source: hello 2.9-2" "$($REPREPRO -b $REPO list buster)"
+}
+
+. shunit2
diff --git a/tests/shunit2-helper-functions.sh b/tests/shunit2-helper-functions.sh
index e99dc972..8f664b8b 100644
--- a/tests/shunit2-helper-functions.sh
+++ b/tests/shunit2-helper-functions.sh
@@ -26,6 +26,7 @@ call() {
 
 check_db() {
 	db_verify $REPO/db/packages.db || fail "BerkeleyDB 'packages.db' is broken."
+	db_verify -o $REPO/db/packagenames.db || fail "BerkeleyDB 'packagenames.db' is broken."
 }
 
 add_distro() {

From 4adbde1341225b8b8abc6b489ebf1bc1705d363a Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 23 Feb 2017 15:48:32 +0100
Subject: [PATCH 51/70] Convert old database format into new format

Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 database.c | 137 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)

diff --git a/database.c b/database.c
index 547fc406..388ff37e 100644
--- a/database.c
+++ b/database.c
@@ -43,6 +43,7 @@
 #include "dpkgversions.h"
 #include "distribution.h"
 #include "database_p.h"
+#include "chunks.h"
 
 #define STRINGIFY(x) #x
 #define TOSTRING(x) STRINGIFY(x)
@@ -1972,6 +1973,130 @@ static int get_package_name(DB *secondary, const DBT *pkey, const DBT *pdata, DB
 	return 0;
 }
 
+static retvalue database_translate_legacy_packages(void) {
+	struct cursor *databases_cursor, *cursor;
+	struct table *legacy_databases, *legacy_table, *packages;
+	const char *chunk, *packagename;
+	char *identifier, *key, *legacy_filename, *packages_filename, *packageversion;
+	retvalue r, result;
+	int ret, e;
+	size_t chunk_len;
+	DBT Key, Data;
+
+	if (verbose >= 15)
+		fprintf(stderr, "trace: database_translate_legacy_packages() called.\n");
+
+	if (!isdir(global.dbdir)) {
+		fprintf(stderr, "Cannot find directory '%s'!\n", global.dbdir);
+		return RET_ERROR;
+	}
+
+	packages_filename = dbfilename("packages.db");
+	legacy_filename = dbfilename("packages.legacy.db");
+	ret = rename(packages_filename, legacy_filename);
+	if (ret != 0) {
+		e = errno;
+		fprintf(stderr, "error %d renaming %s to %s: %s\n",
+				e, packages_filename, legacy_filename, strerror(e));
+		return (e != 0)?e:EINVAL;
+	}
+	if (verbose >= 15)
+		fprintf(stderr, "trace: Moved '%s' to '%s'.\n", packages_filename, legacy_filename);
+
+	r = database_table("packages.legacy.db", NULL, dbt_BTREE, DB_RDONLY, &legacy_databases);
+	assert (r != RET_NOTHING);
+	if (RET_WAS_ERROR(r))
+		return r;
+
+	r = table_newglobalcursor(legacy_databases, true, &databases_cursor);
+	assert (r != RET_NOTHING);
+	if (RET_WAS_ERROR(r)) {
+		(void)table_close(legacy_databases);
+		return r;
+	}
+	result = RET_NOTHING;
+	// Iterate over all databases inside the packages.db file.
+	while (cursor_next(legacy_databases, databases_cursor, &Key, &Data)) {
+		identifier = strndup(Key.data, Key.size);
+		if (FAILEDTOALLOC(identifier)) {
+			RET_UPDATE(result, RET_ERROR_OOM);
+			break;
+		}
+		if (verbose >= 15)
+			fprintf(stderr, "Converting table '%s' to new layout...\n", identifier);
+
+		r = database_table("packages.legacy.db", identifier, dbt_BTREE, DB_RDONLY, &legacy_table);
+		assert (r != RET_NOTHING);
+		if (RET_WAS_ERROR(r)) {
+			free(identifier);
+			RET_UPDATE(result, r);
+			break;
+		}
+
+		r = table_newglobalcursor(legacy_table, true, &cursor);
+		assert (r != RET_NOTHING);
+		if (RET_WAS_ERROR(r)) {
+			(void)table_close(legacy_table);
+			free(identifier);
+			RET_UPDATE(result, r);
+			break;
+		}
+
+		r = database_openpackages(identifier, false, &packages);
+		free(identifier);
+		identifier = NULL;
+		if (RET_WAS_ERROR(r)) {
+			(void)cursor_close(legacy_databases, databases_cursor);
+			(void)table_close(legacy_table);
+			RET_UPDATE(result, r);
+			break;
+		}
+
+		while (cursor_nexttempdata(legacy_table, cursor, &packagename, &chunk, &chunk_len)) {
+			r = chunk_getvalue(chunk, "Version", &packageversion);
+			if (!RET_IS_OK(r)) {
+				RET_UPDATE(result, r);
+				break;
+			}
+			key = package_primarykey(packagename, packageversion);
+			r = table_addrecord(packages, key, chunk, chunk_len, false);
+			free(key);
+			if (RET_WAS_ERROR(r)) {
+				RET_UPDATE(result, r);
+				break;
+			}
+		}
+
+		r = table_close(packages);
+		RET_UPDATE(result, r);
+		r = cursor_close(legacy_table, cursor);
+		RET_UPDATE(result, r);
+		r = table_close(legacy_table);
+		RET_UPDATE(result, r);
+
+		if (RET_WAS_ERROR(result)) {
+			break;
+		}
+		result = RET_OK;
+	}
+	r = cursor_close(legacy_databases, databases_cursor);
+	RET_ENDUPDATE(result, r);
+	r = table_close(legacy_databases);
+	RET_ENDUPDATE(result, r);
+
+	if (RET_IS_OK(result)) {
+		e = deletefile(legacy_filename);
+		if (e != 0) {
+			fprintf(stderr, "Could not delete '%s'!\n"
+"It can now safely be deleted and it all that is left to be done!\n",
+					legacy_filename);
+			return RET_ERRNO(e);
+		}
+	}
+
+	return result;
+}
+
 retvalue database_openpackages(const char *identifier, bool readonly, struct table **table_p) {
 	struct table *table;
 	retvalue r;
@@ -1997,6 +2122,18 @@ retvalue database_openpackages(const char *identifier, bool readonly, struct tab
 	if (RET_WAS_ERROR(r))
 		return r;
 
+	if (table->berkeleydb != NULL && table->sec_berkeleydb == NULL) {
+		r = table_close(table);
+		if (RET_WAS_ERROR(r)) {
+			return r;
+		}
+		r = database_translate_legacy_packages();
+		if (RET_WAS_ERROR(r)) {
+			return r;
+		}
+		return database_openpackages(identifier, readonly, table_p);
+	}
+
 	if (table->berkeleydb != NULL && table->sec_berkeleydb != NULL) {
 		r = table->berkeleydb->associate(table->berkeleydb, NULL,
 				table->sec_berkeleydb, get_package_name, 0);

From a7c605dc009c6db7075037f81ba49f13f4b4acd6 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 27 Aug 2018 14:33:51 +0200
Subject: [PATCH 52/70] [testsuite] Add test case for bug #8

---
 tests/multiversion.sh                   |  14 ++++++++++++++
 tests/old-database/conf/distributions   |   5 +++++
 tests/old-database/db/checksums.db      | Bin 0 -> 16384 bytes
 tests/old-database/db/contents.cache.db | Bin 0 -> 16384 bytes
 tests/old-database/db/packages.db       | Bin 0 -> 57344 bytes
 tests/old-database/db/references.db     | Bin 0 -> 16384 bytes
 tests/old-database/db/release.caches.db | Bin 0 -> 20480 bytes
 tests/old-database/db/version           |   4 ++++
 8 files changed, 23 insertions(+)
 create mode 100644 tests/old-database/conf/distributions
 create mode 100644 tests/old-database/db/checksums.db
 create mode 100644 tests/old-database/db/contents.cache.db
 create mode 100644 tests/old-database/db/packages.db
 create mode 100644 tests/old-database/db/references.db
 create mode 100644 tests/old-database/db/release.caches.db
 create mode 100644 tests/old-database/db/version

diff --git a/tests/multiversion.sh b/tests/multiversion.sh
index 59a995bb..608c6521 100755
--- a/tests/multiversion.sh
+++ b/tests/multiversion.sh
@@ -263,4 +263,18 @@ buster|main|source: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
 	assertEquals "buster|main|source: hello 2.9-2" "$($REPREPRO -b $REPO list buster)"
 }
 
+test_database_upgrade() {
+	# Test case for https://github.com/profitbricks/reprepro/issues/8
+	rm -rf "$REPO"
+	cp -r "${0%/*}/old-database" "$REPO"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO export
+	assertEquals "\
+bullseye|main|amd64
+bullseye|main|i386
+bullseye|main|source
+bullseye|non-free|amd64
+bullseye|non-free|i386
+bullseye|non-free|source" "$(db_dump "$REPO/db/packages.db" | sed -n 's/^database=//p')"
+}
+
 . shunit2
diff --git a/tests/old-database/conf/distributions b/tests/old-database/conf/distributions
new file mode 100644
index 00000000..80ba8587
--- /dev/null
+++ b/tests/old-database/conf/distributions
@@ -0,0 +1,5 @@
+Codename: bullseye
+Architectures: amd64 i386 source
+Components: main non-free
+Log: testrepo.log
+Tracking: all
diff --git a/tests/old-database/db/checksums.db b/tests/old-database/db/checksums.db
new file mode 100644
index 0000000000000000000000000000000000000000..842fdd1a2cd6bcac2a25c5e09bea8f083779ee54
GIT binary patch
literal 16384
zcmeI&KW@S>6bInvK%{mGHz0ebo~Ao<CAbM22jU1w9HSCr$NEJ?{1F2~r7(P!EWcPz
z;(XgrPDDhhJ=^NDjrS}TwGTzaEU&LxU+d#?U&>p=_GkBT_`0lU&g*i`_{{T^fdByl
z1PBlyK!5-N0t5&Um{=gQ`gLA!;?$M)w>x#W^SRBwcKJUgGy((&5FkK+009C72oNCf
zx&red={Hfw&3gSkY&5ffp4~q>{~yov$1mz@@l5`|{1q_XX(T{^009C72oNAZfB*pk
l1f~?o{C}6%7p0wYN8U|<009C72oNAZfB*pk1PJ`6z#kWJ7PJ5W

literal 0
HcmV?d00001

diff --git a/tests/old-database/db/contents.cache.db b/tests/old-database/db/contents.cache.db
new file mode 100644
index 0000000000000000000000000000000000000000..646a2fd1610700b58bf751c6f8a92a64bdb0937c
GIT binary patch
literal 16384
zcmeI&F%E(-6vpvaghZ#3cmTPANAMsnDkdZt4fTX3j_yu|(PP+$mLPF(Fu~#9q~Spe
zl;8HHh=>^9wXNofOteUSZwir-o_AX2>+9@f%vj`nSuCqt+>S=CO6P=Uwu2N1Ab<b@
z2q1s}0tg_000IagFsOiL^=@8|q_!rx+uFWg*F|Q&lmAtMh5!NxAb<b@2q1s}0tg_0
z!2bxuO|p+<DrUXk9je^9e6uaeye!<&J=dw(U-#}m?f-{8{qRMZFTTnD{r?8S{TvZM
z009ILKmY**5I_I{1Q0-A7y-@yy}X{9WSF(^Rs;}0009ILKmY**5I_I{1Q7VWzyoZw
B9X9{~

literal 0
HcmV?d00001

diff --git a/tests/old-database/db/packages.db b/tests/old-database/db/packages.db
new file mode 100644
index 0000000000000000000000000000000000000000..6a7d4985e6bcc42059022d96d984b1b99e4ab83f
GIT binary patch
literal 57344
zcmeI%F>Vt<6b9gNu!9pEO#}$&*b7h~q9_F?-~^N;V5CTiBPB|K<dPF`0!nI55K(aq
zM8zQ}n3u%LWVMA0i%s#3G}6rMtatQH_4}oi(xCKc2QNoyBL>^0Uk*~*ih0$VGXDPj
z$6zo{>G#Foy)SQW9<E&vW6$C@Z=M<vAV7cs0RjXF5FkK+009Dh3Pe_~^ZF$1<iY4q
z{*`~`i~KEr&7bo;zsqm)>%5cY|32@^w+Rp+K!5-N0t5&UAVA>%2@EgO@+3V=gM;&<
zqqFI!>3nuFd-3sfI-Q@LoS(j*rmH=h=}Fg~!&iH|<+xXS#_4I-p8ewwyE}_L>*;Z|
z=Xn2ccKbWS^sxQ>);HGDqxSRb-w(CO{!zQ{s{b!)`mrxsQ~EOX|9i1nBS3%v0RjXF
z5FkK+009C7?u<a>|5je#9*pkH@f?Hz0RjXF5FkK+009C72oP9-K;-{n>0R~zSg%`C
zCd<_S{e4iY851BtfB*pk1PBlyK!5;&{skicxAMCBfB)Cw1Ox~WAV7cs0RjXF5FkKc
zr2>)v*GuoJ|HpdWnlf3Y{(rxI$(R5E0t5&UAV7cs0RjXF5a?YX@_#F@tN-_YD-J+_
z009C72oNAZfB*pk1Xe2$`F~t`SN%WM>(-RXGWGwB`Xyrm1PBlyK!5-N0t5&UAV8pZ
zfyn=@ysrM=`>i+t0RjXF5FkK+009C72oPAUK;-|;(!1*av0k^POqQwtKd4_aCP07y
z0RjXF5FkK+009C7dKZZN-^%Oi|GnRe0}voUfB*pk1PBlyK!5;&)e1!ZpOoHJ|Bv;$
zHD$6){eP=|$(R5E0t5&UAV7cs0RjXF5a?YX@_#F@tN-_YD-J+_009C72oNAZfB*pk
I1Xd^T4@<3XLjV8(

literal 0
HcmV?d00001

diff --git a/tests/old-database/db/references.db b/tests/old-database/db/references.db
new file mode 100644
index 0000000000000000000000000000000000000000..7b8e4f8bb490cf04b85e9d058fe9f368f8dd2658
GIT binary patch
literal 16384
zcmeI&F$%&k7>405)q=XDC(vF&J&{Tr9qLUyl6V0RA$05!%wHO;#lb<S!y5?s8xop6
z+ea!%E_Q4Cb*{M;MeJ=MP3*bX%2?l4SLbHZW&2oe?rFD_b?a=+YqpzYKmY**5I_I{
z1Q0*~0R#|00D(~jEUWkPdZyaDyz#YvpSMGnv6KHzf`$MB2q1s}0tg_000IagfWZF=
zOxuK?X(3mIW2nOE5YF8(E&JQu{ipl?bf=%bXcvob^8fI^fwZ3y0R#|0009ILKmY**
r5I_I{1jZ4t{6EO+h0Dh|3vWdL0R#|0009ILKmY**5I_Kd-wQkeB!m}1

literal 0
HcmV?d00001

diff --git a/tests/old-database/db/release.caches.db b/tests/old-database/db/release.caches.db
new file mode 100644
index 0000000000000000000000000000000000000000..7958f4761be8b23b0915105b74730d344f943283
GIT binary patch
literal 20480
zcmeI3&x;&I6vunkBqTe#JG$W6#)}{%TUD=KRlPlW_7L<ah_8Od#mxq?Mu<obqDQ^w
z=KtYA@Z!Of2N6670gnL@`~!SFn_Xu&@w^jw#Wqtt)7|yvQ}5OG_uI@c48u&esbzb0
zxG{BJm2KG>hAY$lN*ksB#~VM+X4i+|(U-qI+HKd)KHYfMn~piX=GlkZQ36Up2`B+2
zpahhF5>Nt4KnW-TC7=W@mB6IcSM~bF@N_Y|`p@F&;{5wlkfUs>|0fdKlz<XY0!ly$
zC;=s)1eAahPy$Lo39Ki9D@&EXF<c*J`N84gaeLU7flk_gdUnsRU;FR*zdYZc+UYa>
zlf9Gu7wP|t9>T7i@XI$JfB1^LPMbC*pahhF5>Nt4KnW-TC7=Y9fD%vwO5kz{Oxk@_
zqwfws-x#+4T-;vZ;=7%%cmCe~ZTp?=o7<1KKHGXS|6%_2{I&TPn;&h;QQDM%5>Nt4
zKnW-TC7=W@kHBo-?)y+e@X$?Ujy>ewqP3|uav`TsTQ6u*EPmW~`$#EJ&EOLP0b~c%
zZFA02>9KmSm5m9^WGMjpn9$nJIroy1r|QON9GMV`i(Ep-)@7s+OvUIeMdHz!VRm%y
z=w`p)+TQWK2lq?cdw=<IeCy8V!xXTo06hc;#sZR!P%9vGbY2E7R8o%BmWa!MEjjQR
zV|373?9`Zo3zm$*EZ-t}cGgRbJ%E$Yx+^i7+=9v3=E|dW9ME8KCCE6Thhj@XNFGxU
zGT`X#^8vrt4jUi0WwdLIHy<4F{f9RXAcei#Tt4AD?F{nzhs}3u@Xy6v8Hi=7v4!lj
zq>4k7l<tJV*hGeVO(q(elA}R3#d}Ao%dA>Y+;jIO5rv+~c}dddZS%oXCh&zlA(73J
z)-ET+v3grmsdBzC*%{v`8AFs>=}FFRqa^cPJ~+CN$jiKFY#K6qwh756+j1gD%}a#V
zM+mv1ffVx;sFR^R+TuZSyZd0}c#s+^wN=U6R!gh~q?BsR)=4!<CC65b6g8q`?T9T(
zO&~`p4z!_J^eM$8iJm=I=SRS2k*?;%Y8tO6*VSlGGk8h2lS#s*vGs;rB;O$c8To6n
zKINX6oRcMMqT8j6JIfNnAbh!1StUrO7>b1&dvBFxX=(*zjq}*4b0xu9WuZXAl13wQ
zkfJkI)<g7xh)b76kvO80ck2k@Jl&JjeQEHXue9ZVt4aT_{?31Q_(}Btlf}D>TZ_j#
zU+w(0)c<d8f4g;W>-YKh^S9<V=AUkUuqj7rQvymr2`B+2pahhF61W@!layY&yZZl;
z{>yuMs#IFdSj2!3Edwm~^=%SF1xb7gLQG;noJhL>V68Y3Y`cMIY>Vid3k1CuPK2TX
zOK$91H*%R=I(wW92>>I0K`f&yUam~Ncb7})&R{6TWHuxs0CE_CR&@WW4q5Bl`ZK<(
z5uZ~|WUS{>p=88TvQ1IQ28SdLh!H?o*mWQKq@1#AT%1EMMocUmIWaeD4TKV{7(rH?
zoz}%9BAE>WuDw#r%t@>i5wmqHsaYf1ths1RuQ6p|OCV{Pt2j|O`c*sVS<Ekidb)`D
zf`VF4+wBET#i`{cbK$vz%rR7~B8A0(6EmVFka83^0%|h}fTyI8I<qD<aS$<4bnH_m
zFL}cmdu)i%YOUfk+T?>1C7vg5Bl&D?uxu)g<T_>3#Hw%sL-H{P>qKIscU{~*93h^!
aRaPr*{d}t>))$rADctoA`F!e5G5-r~Ym;06

literal 0
HcmV?d00001

diff --git a/tests/old-database/db/version b/tests/old-database/db/version
new file mode 100644
index 00000000..50d2a22f
--- /dev/null
+++ b/tests/old-database/db/version
@@ -0,0 +1,4 @@
+5.2.0
+3.3.0
+bdb5.3.28
+bdb5.3.0

From 84dc701dbe5dfc61f1664a1d240b8958298e233c Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 11 Apr 2017 15:10:22 +0200
Subject: [PATCH 53/70] Use database environment

When opening multiple databases in parallel (needed for the move command
or the archive option), the databases needs to be configured with
locking. Thus an database environment is needed. Open and close the
database environment when getting/releasing the database lock.

Without proper locking, the database might become corrupt:

BDB0689 db/packages.db page 6387 is on free list with type 5
BDB0061 PANIC: Invalid argument
Internal error of the underlying BerkeleyDB database:
Within packages.db subtable jessie-proposed|main|amd64 at put(uniq):
BDB0087 DB_RUNRECOVERY: Fatal error, run database recovery
BDB0060 PANIC: fatal region error detected; run recovery
db_close(packages.db, jessie-proposed|main|amd64): BDB0087
DB_RUNRECOVERY: Fatal error, run database recovery
There have been errors!
---
 database.c | 59 +++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 16 deletions(-)

diff --git a/database.c b/database.c
index 388ff37e..d32bca42 100644
--- a/database.c
+++ b/database.c
@@ -59,6 +59,7 @@ static bool rdb_packagesdatabaseopen;
 static bool rdb_trackingdatabaseopen;
 static /*@null@*/ char *rdb_version, *rdb_lastsupportedversion,
 	*rdb_dbversion, *rdb_lastsupporteddbversion;
+static DB_ENV *rdb_env = NULL;
 
 struct table *rdb_checksums, *rdb_contents;
 struct table *rdb_references;
@@ -84,6 +85,36 @@ static inline char *dbfilename(const char *filename) {
 	return calc_dirconcat(global.dbdir, filename);
 }
 
+static retvalue database_openenv(void) {
+	int dbret;
+
+	dbret = db_env_create(&rdb_env, 0);
+	if (dbret != 0) {
+		fprintf(stderr, "db_env_create: %s\n", db_strerror(dbret));
+		return RET_ERROR;
+	}
+
+	// DB_INIT_LOCK is needed to open multiple databases in one file (e.g. for move command)
+	dbret = rdb_env->open(rdb_env, global.dbdir,
+	                      DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE | DB_INIT_LOCK, 0664);
+	if (dbret != 0) {
+		rdb_env->err(rdb_env, dbret, "environment open: %s", global.dbdir);
+		return RET_ERROR;
+	}
+
+	return RET_OK;
+}
+
+static void database_closeenv(void) {
+	int dbret;
+
+	dbret = rdb_env->close(rdb_env, 0);
+	if (dbret != 0) {
+		fprintf(stderr, "Error: DB_ENV->close: %s\n", db_strerror(dbret));
+	}
+	rdb_env = NULL;
+}
+
 /**********************/
 /* lock file handling */
 /**********************/
@@ -149,6 +180,13 @@ static retvalue database_lock(size_t waitforlock) {
 	}
 	free(lockfile);
 	rdb_locked = true;
+
+	r = database_openenv();
+	if (RET_WAS_ERROR(r)) {
+		(void)unlink(lockfile);
+		free(lockfile);
+		return r;
+	}
 	return RET_OK;
 }
 
@@ -157,6 +195,7 @@ static void releaselock(void) {
 
 	assert (rdb_locked);
 
+	database_closeenv();
 	lockfile = dbfilename("lockfile");
 	if (lockfile == NULL)
 		return;
@@ -230,18 +269,12 @@ static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b);
 #endif
 
 static retvalue database_opentable(const char *filename, /*@null@*/const char *subtable, enum database_type type, uint32_t flags, /*@out@*/DB **result) {
-	char *fullfilename;
 	DB *table;
 	int dbret;
 
-	fullfilename = dbfilename(filename);
-	if (FAILEDTOALLOC(fullfilename))
-		return RET_ERROR_OOM;
-
-	dbret = db_create(&table, NULL, 0);
+	dbret = db_create(&table, rdb_env, 0);
 	if (dbret != 0) {
 		fprintf(stderr, "db_create: %s\n", db_strerror(dbret));
-		free(fullfilename);
 		return RET_DBERR(dbret);
 	}
 	if (type == dbt_BTREEPAIRS || type == dbt_BTREEVERSIONS) {
@@ -249,7 +282,6 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 		if (dbret != 0) {
 			table->err(table, dbret, "db_set_flags(DB_DUPSORT):");
 			(void)table->close(table, 0);
-			free(fullfilename);
 			return RET_DBERR(dbret);
 		}
 	} else if (type == dbt_BTREEDUP) {
@@ -265,7 +297,6 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 		if (dbret != 0) {
 			table->err(table, dbret, "db_set_dup_compare:");
 			(void)table->close(table, 0);
-			free(fullfilename);
 			return RET_DBERR(dbret);
 		}
 	}
@@ -274,7 +305,6 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 		if (dbret != 0) {
 			table->err(table, dbret, "db_set_dup_compare:");
 			(void)table->close(table, 0);
-			free(fullfilename);
 			return RET_DBERR(dbret);
 		}
 	}
@@ -295,24 +325,21 @@ static retvalue database_opentable(const char *filename, /*@null@*/const char *s
 #endif
 #endif
 #endif
-	dbret = DB_OPEN(table, fullfilename, subtable, types[type], flags);
+	dbret = DB_OPEN(table, filename, subtable, types[type], flags);
 	if (dbret == ENOENT && !ISSET(flags, DB_CREATE)) {
 		(void)table->close(table, 0);
-		free(fullfilename);
 		return RET_NOTHING;
 	}
 	if (dbret != 0) {
 		if (subtable != NULL)
 			table->err(table, dbret, "db_open(%s:%s)[%d]",
-					fullfilename, subtable, dbret);
+					filename, subtable, dbret);
 		else
 			table->err(table, dbret, "db_open(%s)[%d]",
-					fullfilename, dbret);
+					filename, dbret);
 		(void)table->close(table, 0);
-		free(fullfilename);
 		return RET_DBERR(dbret);
 	}
-	free(fullfilename);
 	*result = table;
 	return RET_OK;
 }

From 9310facd1345a55a9044410c20d528c42645d059 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 24 Feb 2017 17:09:20 +0100
Subject: [PATCH 54/70] Keep track of all opened database tables

The move command will need to open two tables at the same time (the
source table and the destination table). Thus keep track of all opened
tables.
---
 database.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/database.c b/database.c
index d32bca42..fb803441 100644
--- a/database.c
+++ b/database.c
@@ -67,6 +67,14 @@ static struct {
 	bool createnewtables;
 } rdb_capabilities;
 
+struct opened_tables {
+	struct opened_tables *next;
+	const char *name;
+	const char *subname;
+};
+
+struct opened_tables *opened_tables = NULL;
+
 static void database_free(void) {
 	if (!rdb_initialized)
 		return;
@@ -1044,6 +1052,7 @@ static void table_printerror(struct table *table, int dbret, const char *action)
 }
 
 retvalue table_close(struct table *table) {
+	struct opened_tables *prev = NULL;
 	int dbret;
 	retvalue result = RET_OK;
 
@@ -1071,6 +1080,20 @@ retvalue table_close(struct table *table) {
 				db_strerror(dbret));
 		result = RET_DBERR(dbret);
 	}
+
+	for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) {
+		if(strcmp2(iter->name, table->name) == 0 && strcmp2(iter->subname, table->subname) == 0) {
+			if (prev == NULL) {
+				opened_tables = iter->next;
+			} else {
+				prev->next = iter->next;
+			}
+			free(iter);
+			break;
+		}
+		prev = iter;
+	}
+
 	free(table->name);
 	free(table->subname);
 	free(table);
@@ -1805,8 +1828,19 @@ retvalue database_haspackages(const char *identifier) {
 static retvalue database_table_secondary(const char *filename, const char *subtable, enum database_type type, uint32_t flags,
                                          const char *secondary_filename, enum database_type secondary_type, /*@out@*/struct table **table_p) {
 	struct table *table;
+	struct opened_tables *opened_table;
 	retvalue r;
 
+	for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) {
+		if(strcmp2(iter->name, filename) == 0 && strcmp2(iter->subname, subtable) == 0) {
+            fprintf(stderr,
+ "Internal Error: Trying to open table '%s' from file '%s' multiple times.\n"
+ "This should normally not happen (to avoid triggering bugs in the underlying BerkeleyDB)\n",
+                    subtable, filename);
+			return RET_ERROR;
+		}
+	}
+
 	table = zNEW(struct table);
 	if (FAILEDTOALLOC(table))
 		return RET_ERROR_OOM;
@@ -1876,6 +1910,18 @@ static retvalue database_table_secondary(const char *filename, const char *subta
 		}
 	}
 
+	opened_table = zNEW(struct opened_tables);
+	if (FAILEDTOALLOC(opened_table)) {
+		free(table->subname);
+		free(table->name);
+		free(table);
+		return RET_ERROR_OOM;
+	}
+	opened_table->name = table->name;
+	opened_table->subname = table->subname;
+	opened_table->next = opened_tables;
+	opened_tables = opened_table;
+
 	*table_p = table;
 	return r;
 }

From a359ed285b2f26c374d2c653d02326d3d6ec1286 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 24 Feb 2017 17:11:08 +0100
Subject: [PATCH 55/70] Add print_opened_tables (for debugging purposes)

---
 database.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/database.c b/database.c
index fb803441..8037aaf6 100644
--- a/database.c
+++ b/database.c
@@ -1051,6 +1051,17 @@ static void table_printerror(struct table *table, int dbret, const char *action)
 	}
 }
 
+static void print_opened_tables(FILE *stream) {
+	if (opened_tables == NULL) {
+		fprintf(stream, "No tables are opened.\n");
+	} else {
+		fprintf(stream, "Opened tables:\n");
+		for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) {
+			fprintf(stream, " * %s - '%s'\n", iter->name, iter->subname);
+		}
+	}
+}
+
 retvalue table_close(struct table *table) {
 	struct opened_tables *prev = NULL;
 	int dbret;
@@ -1094,6 +1105,9 @@ retvalue table_close(struct table *table) {
 		prev = iter;
 	}
 
+	if (verbose >= 25)
+		print_opened_tables(stderr);
+
 	free(table->name);
 	free(table->subname);
 	free(table);
@@ -1922,6 +1936,9 @@ static retvalue database_table_secondary(const char *filename, const char *subta
 	opened_table->next = opened_tables;
 	opened_tables = opened_table;
 
+	if (verbose >= 25)
+		print_opened_tables(stderr);
+
 	*table_p = table;
 	return r;
 }

From b4928d668b85ae036d98e281c72232619f90e09a Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 24 Feb 2017 17:15:45 +0100
Subject: [PATCH 56/70] Remove tracking of opened individual databases

---
 database.c | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/database.c b/database.c
index 8037aaf6..49027434 100644
--- a/database.c
+++ b/database.c
@@ -55,8 +55,6 @@
 static bool rdb_initialized, rdb_used, rdb_locked, rdb_verbose;
 static int rdb_dircreationdepth;
 static bool rdb_nopackages, rdb_readonly;
-static bool rdb_packagesdatabaseopen;
-static bool rdb_trackingdatabaseopen;
 static /*@null@*/ char *rdb_version, *rdb_lastsupportedversion,
 	*rdb_dbversion, *rdb_lastsupporteddbversion;
 static DB_ENV *rdb_env = NULL;
@@ -1008,7 +1006,6 @@ struct table {
 	char *name, *subname;
 	DB *berkeleydb;
 	DB *sec_berkeleydb;
-	bool *flagreset;
 	bool readonly, verbose;
 	uint32_t flags;
 };
@@ -1069,8 +1066,6 @@ retvalue table_close(struct table *table) {
 
 	if (table == NULL)
 		return RET_NOTHING;
-	if (table->flagreset != NULL)
-		*table->flagreset = false;
 	if (table->sec_berkeleydb != NULL) {
 		dbret = table->sec_berkeleydb->close(table->sec_berkeleydb, 0);
 		if (dbret != 0) {
@@ -2026,20 +2021,12 @@ retvalue database_opentracking(const char *codename, bool readonly, struct table
 				stderr);
 		return RET_ERROR;
 	}
-	if (rdb_trackingdatabaseopen) {
-		(void)fputs(
-"Internal Error: Trying to open multiple tracking databases at the same time.\nThis should normally not happen (to avoid triggering bugs in the underlying BerkeleyDB)\n",
-				stderr);
-		return RET_ERROR;
-	}
 
 	r = database_table("tracking.db", codename,
 			dbt_BTREEPAIRS, readonly?DB_RDONLY:DB_CREATE, &table);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
 		return r;
-	table->flagreset = &rdb_trackingdatabaseopen;
-	rdb_trackingdatabaseopen = true;
 	*table_p = table;
 	return RET_OK;
 }
@@ -2197,13 +2184,6 @@ retvalue database_openpackages(const char *identifier, bool readonly, struct tab
 				stderr);
 		return RET_ERROR;
 	}
-	if (rdb_packagesdatabaseopen) {
-		(void)fputs(
-"Internal Error: Trying to open multiple packages databases at the same time.\n"
-"This should normally not happen (to avoid triggering bugs in the underlying BerkeleyDB)\n",
-				stderr);
-		return RET_ERROR;
-	}
 
 	r = database_table_secondary("packages.db", identifier,
 			dbt_BTREE, readonly?DB_RDONLY:DB_CREATE,
@@ -2232,8 +2212,6 @@ retvalue database_openpackages(const char *identifier, bool readonly, struct tab
 		}
 	}
 
-	table->flagreset = &rdb_packagesdatabaseopen;
-	rdb_packagesdatabaseopen = true;
 	*table_p = table;
 	return RET_OK;
 }

From f3c21dca25bf3e37cd79179637fffe964ca07083 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 23 Feb 2017 18:52:46 +0100
Subject: [PATCH 57/70] Enhance copy functions parameters to support moving

---
 copypackages.c | 34 ++++++++++++++++++++--------------
 copypackages.h |  8 ++++----
 main.c         |  8 ++++----
 3 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index e3997a89..18c59999 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -256,10 +256,13 @@ static retvalue list_prepareadd(struct package_list *list, struct target *target
 	return RET_OK;
 }
 
-static retvalue package_add(struct distribution *into, /*@null@*/trackingdb tracks, struct target *target, const struct selectedpackage *package, /*@null@*/ const char *suitefrom) {
+static retvalue package_add(struct distribution *into, /*@null@*/trackingdb tracks, struct target *target, const struct selectedpackage *package, /*@null@*/ struct distribution *from, bool remove_source) {
 	struct trackingdata trackingdata;
 	retvalue r;
 
+	// TODO: remove_source = True not implemented yet
+	assert (!remove_source);
+
 	if (verbose >= 1) {
 		printf("Adding '%s' '%s' to '%s'.\n",
 				package->name, package->version,
@@ -286,7 +289,7 @@ static retvalue package_add(struct distribution *into, /*@null@*/trackingdb trac
 			(tracks != NULL)?
 			&trackingdata:NULL,
 			package->architecture,
-			NULL, suitefrom);
+			NULL, from != NULL ? from->codename : NULL);
 	RET_UPDATE(into->status, r);
 	if (tracks != NULL) {
 		retvalue r2;
@@ -297,7 +300,7 @@ static retvalue package_add(struct distribution *into, /*@null@*/trackingdb trac
 	return r;
 }
 
-static retvalue packagelist_add(struct distribution *into, const struct package_list *list, /*@null@*/const char *suitefrom) {
+static retvalue packagelist_add(struct distribution *into, const struct package_list *list, /*@null@*/struct distribution *from, bool remove_source) {
 	retvalue result, r;
 	struct target_package_list *tpl;
 	struct selectedpackage *package;
@@ -325,7 +328,7 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 		for (package = tpl->packages; package != NULL ;
 		                              package = package->next) {
 			r = package_add(into, tracks, target,
-					package, suitefrom);
+					package, from, remove_source);
 			RET_UPDATE(result, r);
 		}
 		r = target_closepackagesdb(target);
@@ -437,7 +440,7 @@ static void packagelist_done(struct package_list *list) {
 	}
 }
 
-retvalue copy_by_name(struct distribution *into, struct distribution *from, struct nameandversion *nameandversion, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes) {
+retvalue copy_by_name(struct distribution *into, struct distribution *from, struct nameandversion *nameandversion, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes, bool remove_source) {
 	struct package_list list;
 	retvalue r;
 
@@ -474,7 +477,7 @@ retvalue copy_by_name(struct distribution *into, struct distribution *from, stru
 	}
 	if (!RET_IS_OK(r))
 		return r;
-	r = packagelist_add(into, &list, from->codename);
+	r = packagelist_add(into, &list, from, remove_source);
 	packagelist_done(&list);
 	return r;
 }
@@ -540,7 +543,7 @@ static retvalue by_source(struct package_list *list, struct target *desttarget,
 	return result;
 }
 
-retvalue copy_by_source(struct distribution *into, struct distribution *from, int argc, const char **argv, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes) {
+retvalue copy_by_source(struct distribution *into, struct distribution *from, int argc, const char **argv, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes, bool remove_source) {
 	struct package_list list;
 	struct namelist names = { argc, argv, NULL, nzNEW(argc, bool) };
 	retvalue r;
@@ -614,7 +617,7 @@ retvalue copy_by_source(struct distribution *into, struct distribution *from, in
 	free(names.found);
 	if (!RET_IS_OK(r))
 		return r;
-	r = packagelist_add(into, &list, from->codename);
+	r = packagelist_add(into, &list, from, remove_source);
 	packagelist_done(&list);
 	return r;
 }
@@ -671,7 +674,7 @@ static retvalue by_glob(struct package_list *list, struct target *desttarget, st
 	return result;
 }
 
-retvalue copy_by_glob(struct distribution *into, struct distribution *from, const char *glob, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes) {
+retvalue copy_by_glob(struct distribution *into, struct distribution *from, const char *glob, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes, bool remove_source) {
 	struct package_list list;
 	retvalue r;
 
@@ -681,12 +684,12 @@ retvalue copy_by_glob(struct distribution *into, struct distribution *from, cons
 			packagetypes, by_glob, (void*)glob);
 	if (!RET_IS_OK(r))
 		return r;
-	r = packagelist_add(into, &list, from->codename);
+	r = packagelist_add(into, &list, from, remove_source);
 	packagelist_done(&list);
 	return r;
 }
 
-retvalue copy_by_formula(struct distribution *into, struct distribution *from, const char *filter, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes) {
+retvalue copy_by_formula(struct distribution *into, struct distribution *from, const char *filter, const struct atomlist *components, const struct atomlist *architectures, const struct atomlist *packagetypes, bool remove_source) {
 	struct package_list list;
 	term *condition;
 	retvalue r;
@@ -702,7 +705,7 @@ retvalue copy_by_formula(struct distribution *into, struct distribution *from, c
 	term_free(condition);
 	if (!RET_IS_OK(r))
 		return r;
-	r = packagelist_add(into, &list, from->codename);
+	r = packagelist_add(into, &list, from, remove_source);
 	packagelist_done(&list);
 	return r;
 }
@@ -832,7 +835,7 @@ retvalue copy_from_file(struct distribution *into, component_t component, archit
 	r = indexfile_close(i);
 	RET_ENDUPDATE(result, r);
 	if (RET_IS_OK(result))
-		result = packagelist_add(into, &list, NULL);
+		result = packagelist_add(into, &list, NULL, false);
 	packagelist_done(&list);
 	return result;
 }
@@ -845,6 +848,7 @@ static retvalue restore_from_snapshot(struct distribution *into, const struct at
 	struct target *target;
 	char *basedir;
 	enum compression compression;
+	struct distribution pseudo_from; // just stores the codename
 
 	basedir = calc_snapshotbasedir(into->codename, snapshotname);
 	if (FAILEDTOALLOC(basedir))
@@ -926,7 +930,9 @@ static retvalue restore_from_snapshot(struct distribution *into, const struct at
 	free(basedir);
 	if (RET_WAS_ERROR(result))
 		return result;
-	r = packagelist_add(into, &list, snapshotname);
+	memset(&pseudo_from, 0, sizeof(struct distribution));
+	pseudo_from.codename = (char*)snapshotname;
+	r = packagelist_add(into, &list, &pseudo_from, false);
 	packagelist_done(&list);
 	return r;
 }
diff --git a/copypackages.h b/copypackages.h
index 57682065..0f4771ff 100644
--- a/copypackages.h
+++ b/copypackages.h
@@ -12,10 +12,10 @@ struct nameandversion {
 	bool found;
 };
 
-retvalue copy_by_name(struct distribution * /*into*/, struct distribution * /*from*/, struct nameandversion *, const struct atomlist *, const struct atomlist *, const struct atomlist *);
-retvalue copy_by_source(struct distribution * /*into*/, struct distribution * /*from*/, int, const char **, const struct atomlist *, const struct atomlist *, const struct atomlist *);
-retvalue copy_by_formula(struct distribution * /*into*/, struct distribution * /*from*/, const char *formula, const struct atomlist *, const struct atomlist *, const struct atomlist *);
-retvalue copy_by_glob(struct distribution * /*into*/, struct distribution * /*from*/, const char * /*glob*/, const struct atomlist *, const struct atomlist *, const struct atomlist *);
+retvalue copy_by_name(struct distribution * /*into*/, struct distribution * /*from*/, struct nameandversion *, const struct atomlist *, const struct atomlist *, const struct atomlist *, bool);
+retvalue copy_by_source(struct distribution * /*into*/, struct distribution * /*from*/, int, const char **, const struct atomlist *, const struct atomlist *, const struct atomlist *, bool);
+retvalue copy_by_formula(struct distribution * /*into*/, struct distribution * /*from*/, const char *formula, const struct atomlist *, const struct atomlist *, const struct atomlist *, bool);
+retvalue copy_by_glob(struct distribution * /*into*/, struct distribution * /*from*/, const char * /*glob*/, const struct atomlist *, const struct atomlist *, const struct atomlist *, bool);
 
 retvalue copy_from_file(struct distribution * /*into*/, component_t, architecture_t, packagetype_t, const char * /*filename*/ , int, const char **);
 
diff --git a/main.c b/main.c
index d2cf1463..f6526f1b 100644
--- a/main.c
+++ b/main.c
@@ -2006,7 +2006,7 @@ ACTION_D(y, n, y, copy) {
 	data[i].version = NULL;
 
 	result = copy_by_name(destination, source, data,
-			components, architectures, packagetypes);
+			components, architectures, packagetypes, false);
 	for (i = 0; i < argc - 3; i++) {
 		splitnameandversion_done(&data[i].name, &data[i].version);
 	}
@@ -2036,7 +2036,7 @@ ACTION_D(y, n, y, copysrc) {
 		return result;
 
 	return copy_by_source(destination, source, argc-3, argv+3,
-			components, architectures, packagetypes);
+			components, architectures, packagetypes, false);
 	return result;
 }
 
@@ -2065,7 +2065,7 @@ ACTION_D(y, n, y, copyfilter) {
 		return result;
 
 	return copy_by_formula(destination, source, argv[3],
-			components, architectures, packagetypes);
+			components, architectures, packagetypes, false);
 }
 
 ACTION_D(y, n, y, copymatched) {
@@ -2093,7 +2093,7 @@ ACTION_D(y, n, y, copymatched) {
 		return result;
 
 	return copy_by_glob(destination, source, argv[3],
-			components, architectures, packagetypes);
+			components, architectures, packagetypes, false);
 }
 
 ACTION_D(y, n, y, restore) {

From 801a6ddab0f9159dedafe8e55e587423d1923480 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 23 Feb 2017 19:22:21 +0100
Subject: [PATCH 58/70] Add fromtarget to struct target_package_list

The move commands needs access to the from target to remove the packages
after adding them to the destination target.
---
 copypackages.c | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index 18c59999..ea88a576 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -42,6 +42,7 @@
 struct target_package_list {
 	struct target_package_list *next;
 	struct target *target;
+	struct target *fromtarget;
 	struct selectedpackage {
 		/*@null@*/struct selectedpackage *next;
 		char *name;
@@ -71,19 +72,20 @@ static int cascade_strcmp(const char *s1, const char *s2, const char *t1, const
 	return result;
 }
 
-static retvalue list_newpackage(struct package_list *list, struct target *target, const char *sourcename, const char *sourceversion, const char *packagename, const char *packageversion, /*@out@*/struct selectedpackage **package_p) {
+static retvalue list_newpackage(struct package_list *list, struct target *desttarget, struct target *fromtarget, const char *sourcename, const char *sourceversion, const char *packagename, const char *packageversion, /*@out@*/struct selectedpackage **package_p) {
 	struct target_package_list *t, **t_p;
 	struct selectedpackage *package, **p_p;
 	int c;
 
 	t_p = &list->targets;
-	while (*t_p != NULL && (*t_p)->target != target)
+	while (*t_p != NULL && (*t_p)->target != desttarget && (*t_p)->fromtarget != fromtarget)
 		t_p = &(*t_p)->next;
 	if (*t_p == NULL) {
 		t = zNEW(struct target_package_list);
 		if (FAILEDTOALLOC(t))
 			return RET_ERROR_OOM;
-		t->target = target;
+		t->target = desttarget;
+		t->fromtarget = fromtarget;
 		t->next = *t_p;
 		*t_p = t;
 	} else
@@ -166,12 +168,12 @@ static void list_cancelpackage(struct package_list *list, /*@only@*/struct selec
 	assert (package == NULL);
 }
 
-static retvalue list_prepareadd(struct package_list *list, struct target *target, struct package *package) {
+static retvalue list_prepareadd(struct package_list *list, struct target *desttarget, struct target *fromtarget, struct package *package) {
 	struct selectedpackage *new SETBUTNOTUSED(= NULL);
 	retvalue r;
 	int i;
 
-	assert (target->packagetype == package->target->packagetype);
+	assert (desttarget->packagetype == package->target->packagetype);
 
 	r = package_getversion(package);
 	assert (r != RET_NOTHING);
@@ -185,7 +187,7 @@ static retvalue list_prepareadd(struct package_list *list, struct target *target
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r))
 		return r;
-	r = list_newpackage(list, target,
+	r = list_newpackage(list, desttarget, fromtarget,
 			package->source, package->sourceversion,
 			package->name, package->version, &new);
 	assert (r != RET_NOTHING);
@@ -194,7 +196,7 @@ static retvalue list_prepareadd(struct package_list *list, struct target *target
 	assert (new != NULL);
 
 	new->architecture = package->architecture;
-	r = target->getinstalldata(target, package,
+	r = desttarget->getinstalldata(desttarget, package,
 			&new->control, &new->filekeys, &new->origfiles);
 	assert (r != RET_NOTHING);
 	if (RET_WAS_ERROR(r)) {
@@ -256,7 +258,7 @@ static retvalue list_prepareadd(struct package_list *list, struct target *target
 	return RET_OK;
 }
 
-static retvalue package_add(struct distribution *into, /*@null@*/trackingdb tracks, struct target *target, const struct selectedpackage *package, /*@null@*/ struct distribution *from, bool remove_source) {
+static retvalue package_add(struct distribution *into, /*@null@*/trackingdb tracks, struct target *target, const struct selectedpackage *package, /*@null@*/ struct distribution *from, struct target *fromtarget, bool remove_source) {
 	struct trackingdata trackingdata;
 	retvalue r;
 
@@ -320,6 +322,7 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 	result = RET_NOTHING;
 	for (tpl = list->targets; tpl != NULL ; tpl = tpl->next) {
 		struct target *target = tpl->target;
+		struct target *fromtarget = tpl->fromtarget;
 
 		r = target_initpackagesdb(target, READWRITE);
 		RET_ENDUPDATE(result, r);
@@ -328,7 +331,7 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 		for (package = tpl->packages; package != NULL ;
 		                              package = package->next) {
 			r = package_add(into, tracks, target,
-					package, from, remove_source);
+					package, from, fromtarget, remove_source);
 			RET_UPDATE(result, r);
 		}
 		r = target_closepackagesdb(target);
@@ -414,7 +417,7 @@ static retvalue by_name(struct package_list *list, struct target *desttarget, st
 		RET_ENDUPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			break;
-		r = list_prepareadd(list, desttarget, &package);
+		r = list_prepareadd(list, desttarget, fromtarget, &package);
 		package_done(&package);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
@@ -531,7 +534,7 @@ static retvalue by_source(struct package_list *list, struct target *desttarget,
 				continue;
 			}
 		}
-		r = list_prepareadd(list, desttarget, &iterator.current);
+		r = list_prepareadd(list, desttarget, fromtarget, &iterator.current);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			break;
@@ -641,7 +644,7 @@ static retvalue by_formula(struct package_list *list, struct target *desttarget,
 			result = r;
 			break;
 		}
-		r = list_prepareadd(list, desttarget, &iterator.current);
+		r = list_prepareadd(list, desttarget, fromtarget, &iterator.current);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			break;
@@ -664,7 +667,7 @@ static retvalue by_glob(struct package_list *list, struct target *desttarget, st
 	while (package_next(&iterator)) {
 		if (!globmatch(iterator.current.name, glob))
 			continue;
-		r = list_prepareadd(list, desttarget, &iterator.current);
+		r = list_prepareadd(list, desttarget, fromtarget, &iterator.current);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			break;
@@ -826,7 +829,7 @@ retvalue copy_from_file(struct distribution *into, component_t component, archit
 	while (indexfile_getnext(i, &package, target, false)) {
 		r = choose_by_name(&package, &d);
 		if (RET_IS_OK(r))
-			r = list_prepareadd(&list, target, &package);
+			r = list_prepareadd(&list, target, NULL, &package);
 		package_done(&package);
 		RET_UPDATE(result, r);
 		if (RET_WAS_ERROR(result))
@@ -916,7 +919,7 @@ static retvalue restore_from_snapshot(struct distribution *into, const struct at
 			result = action(&package, d);
 			if (RET_IS_OK(result))
 				result = list_prepareadd(&list,
-						target, &package);
+						target, NULL, &package);
 			package_done(&package);
 			if (RET_WAS_ERROR(result))
 				break;

From ee6e1779f1a7aebe00cd8371ff60d1251f575fe7 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 30 Mar 2017 14:49:59 +0200
Subject: [PATCH 59/70] package_add: Add fromtracks parameter

---
 copypackages.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index ea88a576..dfa127c1 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -258,7 +258,7 @@ static retvalue list_prepareadd(struct package_list *list, struct target *destta
 	return RET_OK;
 }
 
-static retvalue package_add(struct distribution *into, /*@null@*/trackingdb tracks, struct target *target, const struct selectedpackage *package, /*@null@*/ struct distribution *from, struct target *fromtarget, bool remove_source) {
+static retvalue package_add(struct distribution *into, /*@null@*/trackingdb tracks, struct target *target, const struct selectedpackage *package, /*@null@*/ struct distribution *from, /*@null@*/trackingdb fromtracks, struct target *fromtarget, bool remove_source) {
 	struct trackingdata trackingdata;
 	retvalue r;
 
@@ -306,7 +306,7 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 	retvalue result, r;
 	struct target_package_list *tpl;
 	struct selectedpackage *package;
-	trackingdb tracks;
+	trackingdb tracks, fromtracks = NULL;
 
 	r = distribution_prepareforwriting(into);
 	if (RET_WAS_ERROR(r))
@@ -331,7 +331,7 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 		for (package = tpl->packages; package != NULL ;
 		                              package = package->next) {
 			r = package_add(into, tracks, target,
-					package, from, fromtarget, remove_source);
+					package, from, fromtracks, fromtarget, remove_source);
 			RET_UPDATE(result, r);
 		}
 		r = target_closepackagesdb(target);

From e2d15aa49b5162c17650b990cdac103d3f4573d4 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 23 Feb 2017 19:26:52 +0100
Subject: [PATCH 60/70] Implement remove source support

---
 copypackages.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/copypackages.c b/copypackages.c
index dfa127c1..ae2f4f94 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -262,9 +262,6 @@ static retvalue package_add(struct distribution *into, /*@null@*/trackingdb trac
 	struct trackingdata trackingdata;
 	retvalue r;
 
-	// TODO: remove_source = True not implemented yet
-	assert (!remove_source);
-
 	if (verbose >= 1) {
 		printf("Adding '%s' '%s' to '%s'.\n",
 				package->name, package->version,
@@ -293,12 +290,33 @@ static retvalue package_add(struct distribution *into, /*@null@*/trackingdb trac
 			package->architecture,
 			NULL, from != NULL ? from->codename : NULL);
 	RET_UPDATE(into->status, r);
+
 	if (tracks != NULL) {
 		retvalue r2;
 
 		r2 = trackingdata_finish(tracks, &trackingdata);
 		RET_ENDUPDATE(r, r2);
 	}
+
+	if (!RET_WAS_ERROR(r) && remove_source) {
+		if (fromtracks != NULL) {
+			r = trackingdata_summon(fromtracks, package->sourcename,
+					package->version, &trackingdata);
+			if (RET_WAS_ERROR(r))
+				return r;
+		}
+		r = target_removepackage(fromtarget,
+				from->logger,
+				package->name, package->version,
+				(tracks != NULL) ? &trackingdata : NULL);
+		RET_UPDATE(from->status, r);
+		if (fromtracks != NULL) {
+			retvalue r2;
+
+			r2 = trackingdata_finish(fromtracks, &trackingdata);
+			RET_ENDUPDATE(r, r2);
+		}
+	}
 	return r;
 }
 
@@ -312,6 +330,12 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 	if (RET_WAS_ERROR(r))
 		return r;
 
+	if (remove_source) {
+		r = distribution_prepareforwriting(from);
+		if (RET_WAS_ERROR(r))
+			return r;
+	}
+
 	if (into->tracking != dt_NONE) {
 		r = tracking_initialize(&tracks, into, false);
 		if (RET_WAS_ERROR(r))
@@ -319,6 +343,12 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 	} else
 		tracks = NULL;
 
+	if (from->tracking != dt_NONE) {
+		r = tracking_initialize(&fromtracks, from, false);
+		if (RET_WAS_ERROR(r))
+			return r;
+	}
+
 	result = RET_NOTHING;
 	for (tpl = list->targets; tpl != NULL ; tpl = tpl->next) {
 		struct target *target = tpl->target;
@@ -328,16 +358,33 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 		RET_ENDUPDATE(result, r);
 		if (RET_WAS_ERROR(r))
 			break;
+
+		if (remove_source) {
+			r = target_initpackagesdb(fromtarget, READWRITE);
+			RET_ENDUPDATE(result, r);
+			if (RET_WAS_ERROR(r)) {
+				(void)target_closepackagesdb(target);
+				break;
+			}
+		}
+
 		for (package = tpl->packages; package != NULL ;
 		                              package = package->next) {
 			r = package_add(into, tracks, target,
 					package, from, fromtracks, fromtarget, remove_source);
 			RET_UPDATE(result, r);
 		}
+		if (remove_source) {
+			r = target_closepackagesdb(fromtarget);
+			RET_UPDATE(into->status, r);
+			RET_ENDUPDATE(result, r);
+		}
 		r = target_closepackagesdb(target);
 		RET_UPDATE(into->status, r);
 		RET_ENDUPDATE(result, r);
 	}
+	r = tracking_done(fromtracks, from);
+	RET_ENDUPDATE(result, r);
 	r = tracking_done(tracks, into);
 	RET_ENDUPDATE(result, r);
 	return result;

From 7510b360cc5d4795d972951514ef9d7a01b14422 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Thu, 23 Feb 2017 18:53:58 +0100
Subject: [PATCH 61/70] Add move* commands

Add the commands move, movesrc, movematched, movefilter.
---
 docs/reprepro.1               | 34 ++++++++++++++++
 docs/reprepro.bash_completion |  6 ++-
 main.c                        | 76 +++++++++++++++++++++++++++--------
 3 files changed, 99 insertions(+), 17 deletions(-)

diff --git a/docs/reprepro.1 b/docs/reprepro.1
index a9d3aec3..ac739782 100644
--- a/docs/reprepro.1
+++ b/docs/reprepro.1
@@ -919,6 +919,40 @@ The packages are copied verbatim, no override files are consulted.
 Only components and architectures present in the source distribution are
 copied.
 .TP
+.B move \fIdestination-codename\fP \fIsource-codename\fP \fIpackage\fP\fR[\fP=\fIversion\fP\fR]\fP \fI...\fP
+Move the given packages from one distribution to another.
+The packages are moved verbatim, no override files are consulted.
+Only components and architectures present in the source distribution are
+moved. Package versions must be specified by appending '\fB=\fP' and the
+version to the name (without spaces). When no version is specified, the latest
+package version is moved.
+.TP
+.B movesrc \fIdestination-codename\fP \fIsource-codename\fP \fIsource-package\fP \fR[\fP\fIversions\fP\fR]\fP
+look at each package
+(where package means, as usual, every package be it dsc, deb or udeb)
+in the distribution specified by \fIsource-codename\fP
+and identifies the relevant source package for each.
+All packages matching the specified \fIsource-package\fP name
+(and any \fIversion\fP if specified)
+are moved to the \fIdestination-codename\fP distribution.
+The packages are moved verbatim, no override files are consulted.
+Only components and architectures present in the source distribution are
+moved.
+.TP
+.B movematched \fIdestination-codename\fP \fIsource-codename\fP \fIglob\fP
+Move packages matching the given glob (see \fBlistmatched\fP).
+
+The packages are moved verbatim, no override files are consulted.
+Only components and architectures present in the source distribution are
+moved.
+.TP
+.B movefilter \fIdestination-codename\fP \fIsource-codename\fP \fIformula\fP
+Move packages matching the given formula (see \fBlistfilter\fP).
+(all versions if no version is specified).
+The packages are moved verbatim, no override files are consulted.
+Only components and architectures present in the source distribution are
+moved.
+.TP
 .B restore \fIcodename\fP \fIsnapshot\fP \fIpackages...\fP
 .TP
 .B restoresrc \fIcodename\fP \fIsnapshot\fP \fIsource-epackage\fP \fR[\fP\fIversions\fP\fR]\fP
diff --git a/docs/reprepro.bash_completion b/docs/reprepro.bash_completion
index 7872c155..508a651e 100644
--- a/docs/reprepro.bash_completion
+++ b/docs/reprepro.bash_completion
@@ -310,6 +310,10 @@ _reprepro()
 			listmatched\
 			ls\
 			lsbycomponent\
+			move\
+			movefilter\
+			movematched\
+			movesrc\
 			predelete\
 			processincoming\
 			pull\
@@ -539,7 +543,7 @@ _reprepro()
 			fi
 			return 0;
 			;;
-		copy|copysrc|copyfilter|copymatched)
+		copy|copysrc|copyfilter|copymatched|move|movesrc|movefilter|movematched)
 			# first argument is a codename
 			if [[ $i -eq $COMP_CWORD ]] ; then
 				parse_config
diff --git a/main.c b/main.c
index f6526f1b..f0a28163 100644
--- a/main.c
+++ b/main.c
@@ -1968,7 +1968,8 @@ ACTION_B(y, n, y, dumppull) {
 	return result;
 }
 
-ACTION_D(y, n, y, copy) {
+static retvalue copy_or_move(struct distribution *alldistributions, const struct atomlist * architectures, const struct atomlist * components,
+                             const struct atomlist * packagetypes, int argc, const char * argv[], bool remove_source) {
 	struct distribution *destination, *source;
 	struct nameandversion data[argc-2];
 	int i;
@@ -1985,8 +1986,8 @@ ACTION_D(y, n, y, copy) {
 
 	if (destination->readonly) {
 		fprintf(stderr,
-"Cannot copy packages to read-only distribution '%s'.\n",
-				destination->codename);
+"Cannot %s packages to read-only distribution '%s'.\n",
+				remove_source ? "move" : "copy", destination->codename);
 		return RET_ERROR;
 	}
 	result = distribution_prepareforwriting(destination);
@@ -2006,14 +2007,23 @@ ACTION_D(y, n, y, copy) {
 	data[i].version = NULL;
 
 	result = copy_by_name(destination, source, data,
-			components, architectures, packagetypes, false);
+			components, architectures, packagetypes, remove_source);
 	for (i = 0; i < argc - 3; i++) {
 		splitnameandversion_done(&data[i].name, &data[i].version);
 	}
 	return result;
 }
 
-ACTION_D(y, n, y, copysrc) {
+ACTION_D(y, n, y, copy) {
+	return copy_or_move(alldistributions, architectures, components, packagetypes, argc, argv, false);
+}
+
+ACTION_D(y, n, y, move) {
+	return copy_or_move(alldistributions, architectures, components, packagetypes, argc, argv, true);
+}
+
+static retvalue copysrc_or_movesrc(struct distribution *alldistributions, const struct atomlist * architectures, const struct atomlist * components,
+                                   const struct atomlist * packagetypes, int argc, const char * argv[], bool remove_source) {
 	struct distribution *destination, *source;
 	retvalue result;
 
@@ -2027,8 +2037,8 @@ ACTION_D(y, n, y, copysrc) {
 		return result;
 	if (destination->readonly) {
 		fprintf(stderr,
-"Cannot copy packages to read-only distribution '%s'.\n",
-				destination->codename);
+"Cannot %s packages to read-only distribution '%s'.\n",
+				remove_source ? "move" : "copy", destination->codename);
 		return RET_ERROR;
 	}
 	result = distribution_prepareforwriting(destination);
@@ -2036,11 +2046,20 @@ ACTION_D(y, n, y, copysrc) {
 		return result;
 
 	return copy_by_source(destination, source, argc-3, argv+3,
-			components, architectures, packagetypes, false);
+			components, architectures, packagetypes, remove_source);
 	return result;
 }
 
-ACTION_D(y, n, y, copyfilter) {
+ACTION_D(y, n, y, copysrc) {
+	return copysrc_or_movesrc(alldistributions, architectures, components, packagetypes, argc, argv, false);
+}
+
+ACTION_D(y, n, y, movesrc) {
+	return copysrc_or_movesrc(alldistributions, architectures, components, packagetypes, argc, argv, true);
+}
+
+static retvalue copy_or_move_filter(struct distribution *alldistributions, const struct atomlist * architectures, const struct atomlist * components,
+                                    const struct atomlist * packagetypes, int argc, const char * argv[], bool remove_source) {
 	struct distribution *destination, *source;
 	retvalue result;
 
@@ -2056,8 +2075,8 @@ ACTION_D(y, n, y, copyfilter) {
 		return result;
 	if (destination->readonly) {
 		fprintf(stderr,
-"Cannot copy packages to read-only distribution '%s'.\n",
-				destination->codename);
+"Cannot %s packages to read-only distribution '%s'.\n",
+				remove_source ? "move" : "copy", destination->codename);
 		return RET_ERROR;
 	}
 	result = distribution_prepareforwriting(destination);
@@ -2065,10 +2084,19 @@ ACTION_D(y, n, y, copyfilter) {
 		return result;
 
 	return copy_by_formula(destination, source, argv[3],
-			components, architectures, packagetypes, false);
+			components, architectures, packagetypes, remove_source);
 }
 
-ACTION_D(y, n, y, copymatched) {
+ACTION_D(y, n, y, copyfilter) {
+	return copy_or_move_filter(alldistributions, architectures, components, packagetypes, argc, argv, false);
+}
+
+ACTION_D(y, n, y, movefilter) {
+	return copy_or_move_filter(alldistributions, architectures, components, packagetypes, argc, argv, true);
+}
+
+static retvalue copy_or_move_matched(struct distribution *alldistributions, const struct atomlist * architectures, const struct atomlist * components,
+                                     const struct atomlist * packagetypes, int argc, const char * argv[], bool remove_source) {
 	struct distribution *destination, *source;
 	retvalue result;
 
@@ -2084,8 +2112,8 @@ ACTION_D(y, n, y, copymatched) {
 		return result;
 	if (destination->readonly) {
 		fprintf(stderr,
-"Cannot copy packages to read-only distribution '%s'.\n",
-				destination->codename);
+"Cannot %s packages to read-only distribution '%s'.\n",
+				remove_source ? "move" : "copy", destination->codename);
 		return RET_ERROR;
 	}
 	result = distribution_prepareforwriting(destination);
@@ -2093,7 +2121,15 @@ ACTION_D(y, n, y, copymatched) {
 		return result;
 
 	return copy_by_glob(destination, source, argv[3],
-			components, architectures, packagetypes, false);
+			components, architectures, packagetypes, remove_source);
+}
+
+ACTION_D(y, n, y, copymatched) {
+	return copy_or_move_matched(alldistributions, architectures, components, packagetypes, argc, argv, false);
+}
+
+ACTION_D(y, n, y, movematched) {
+	return copy_or_move_matched(alldistributions, architectures, components, packagetypes, argc, argv, true);
 }
 
 ACTION_D(y, n, y, restore) {
@@ -4043,6 +4079,14 @@ static const struct action {
 		3, 3, "[-C <component> ] [-A <architecture>] [-T <packagetype>] copymatched <destination-distribution> <source-distribution> <glob>"},
 	{"copyfilter",		A_Dact(copyfilter),
 		3, 3, "[-C <component> ] [-A <architecture>] [-T <packagetype>] copyfilter <destination-distribution> <source-distribution> <formula>"},
+	{"move",		A_Dact(move),
+		3, -1, "[-C <component> ] [-A <architecture>] [-T <packagetype>] move <destination-distribution> <source-distribution> <package-names to move>"},
+	{"movesrc",		A_Dact(movesrc),
+		3, -1, "[-C <component> ] [-A <architecture>] [-T <packagetype>] movesrc <destination-distribution> <source-distribution> <source-package-name> [<source versions>]"},
+	{"movematched",		A_Dact(movematched),
+		3, 3, "[-C <component> ] [-A <architecture>] [-T <packagetype>] movematched <destination-distribution> <source-distribution> <glob>"},
+	{"movefilter",		A_Dact(movefilter),
+		3, 3, "[-C <component> ] [-A <architecture>] [-T <packagetype>] movefilter <destination-distribution> <source-distribution> <formula>"},
 	{"restore",		A_Dact(restore),
 		3, -1, "[-C <component> ] [-A <architecture>] [-T <packagetype>] restore <distribution> <snapshot-name> <package-names to restore>"},
 	{"restoresrc",		A_Dact(restoresrc),

From 1d771a59ea5d558ee324ef513cde594cc4177918 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 27 Feb 2017 14:13:04 +0100
Subject: [PATCH 62/70] [testsuite] Add test cases for move* command

Add test cases for the move, movesrc, movematched, movefilter commands.

Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 tests/basic.sh        | 81 +++++++++++++++++++++++++++++++++++++++++++
 tests/multiversion.sh | 37 ++++++++++++++++++++
 2 files changed, 118 insertions(+)

diff --git a/tests/basic.sh b/tests/basic.sh
index 8e1f4a47..4cd5f187 100755
--- a/tests/basic.sh
+++ b/tests/basic.sh
@@ -158,4 +158,85 @@ buster
 bullseye" "$($REPREPRO -b $REPO _listcodenames)"
 }
 
+test_copysrc() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
+	add_distro bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copysrc bullseye buster sl
+	assertEquals "\
+bullseye|main|$ARCH: sl 3.03-1
+bullseye|main|$ARCH: sl-addons 3.03-1
+bullseye|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list bullseye)"
+	assertEquals "\
+buster|main|$ARCH: sl 3.03-1
+buster|main|$ARCH: sl-addons 3.03-1
+buster|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_copymatched() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
+	add_distro bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO copymatched bullseye buster "sl-a*on?"
+	assertEquals "bullseye|main|$ARCH: sl-addons 3.03-1" "$($REPREPRO -b $REPO list bullseye)"
+	assertEquals "\
+buster|main|$ARCH: sl 3.03-1
+buster|main|$ARCH: sl-addons 3.03-1
+buster|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_move() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedsc buster $PKGS/sl_3.03-1.dsc
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/sl_3.03-1_$ARCH.deb $PKGS/sl-addons_3.03-1_all.deb
+	add_distro bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO move bullseye buster sl
+	assertEquals "\
+bullseye|main|$ARCH: sl 3.03-1
+bullseye|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list bullseye)"
+	assertEquals "buster|main|$ARCH: sl-addons 3.03-1" "$($REPREPRO -b $REPO list buster)"
+	assertEquals "\
+Distribution: buster
+Source: sl
+Version: 3.03-1
+Files:
+ pool/main/s/sl/sl_3.03-1.dsc s 0
+ pool/main/s/sl/sl_3.03.orig.tar.gz s 0
+ pool/main/s/sl/sl_3.03-1.debian.tar.xz s 0
+ pool/main/s/sl/sl_3.03-1_$ARCH.deb b 0
+ pool/main/s/sl/sl-addons_3.03-1_all.deb a 1
+
+Distribution: bullseye
+Source: sl
+Version: 3.03-1
+Files:
+ pool/main/s/sl/sl_3.03-1_$ARCH.deb b 1
+ pool/main/s/sl/sl_3.03-1.dsc s 1
+ pool/main/s/sl/sl_3.03.orig.tar.gz s 1
+ pool/main/s/sl/sl_3.03-1.debian.tar.xz s 1" "$($REPREPRO -b $REPO dumptracks)"
+}
+
+test_movesrc() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
+	add_distro bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO movesrc bullseye buster sl
+	assertEquals "\
+bullseye|main|$ARCH: sl 3.03-1
+bullseye|main|$ARCH: sl-addons 3.03-1
+bullseye|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list bullseye)"
+	assertEquals "" "$($REPREPRO -b $REPO list buster)"
+}
+
+test_movematched() {
+	(cd $PKGS && PACKAGE=sl SECTION=main DISTRI=buster EPOCH="" VERSION=3.03 REVISION=-1 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/sl_3.03-1_${ARCH}.changes
+	add_distro bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO movematched bullseye buster "sl-a*on?"
+	assertEquals "bullseye|main|$ARCH: sl-addons 3.03-1" "$($REPREPRO -b $REPO list bullseye)"
+	assertEquals "\
+buster|main|$ARCH: sl 3.03-1
+buster|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list buster)"
+}
+
 . shunit2
diff --git a/tests/multiversion.sh b/tests/multiversion.sh
index 608c6521..4562a9d3 100755
--- a/tests/multiversion.sh
+++ b/tests/multiversion.sh
@@ -277,4 +277,41 @@ bullseye|non-free|i386
 bullseye|non-free|source" "$(db_dump "$REPO/db/packages.db" | sed -n 's/^database=//p')"
 }
 
+test_move_specific() {
+	four_hellos
+	add_distro bullseye
+	$REPREPRO -b $REPO export bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO move bullseye buster hello=2.9-2
+	assertEquals "\
+buster|main|$ARCH: hello 2.9-10
+buster|main|$ARCH: hello 2.9-2+deb8u1
+buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	assertEquals "bullseye|main|$ARCH: hello 2.9-2" "$($REPREPRO -b $REPO list bullseye)"
+}
+
+test_movesrc_specific() {
+	four_hellos
+	add_distro bullseye
+	$REPREPRO -b $REPO export bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO movesrc bullseye buster hello 2.9-2
+	assertEquals "\
+buster|main|$ARCH: hello 2.9-10
+buster|main|$ARCH: hello 2.9-2+deb8u1
+buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	assertEquals "bullseye|main|$ARCH: hello 2.9-2" "$($REPREPRO -b $REPO list bullseye)"
+}
+
+test_movefilter_specific() {
+	four_hellos
+	add_distro bullseye "Limit: -1"
+	$REPREPRO -b $REPO export bullseye
+	call $REPREPRO $VERBOSE_ARGS -b $REPO movefilter bullseye buster 'Package (= hello), $Version (>> 2.9-2)'
+	assertEquals "\
+buster|main|$ARCH: hello 2.9-2
+buster|main|$ARCH: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	assertEquals "\
+bullseye|main|$ARCH: hello 2.9-10
+bullseye|main|$ARCH: hello 2.9-2+deb8u1" "$($REPREPRO -b $REPO list bullseye)"
+}
+
 . shunit2

From e9161a20e478d0fe6d8ddb81f70cf5b70b20d9a2 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 28 Feb 2017 18:13:40 +0100
Subject: [PATCH 63/70] Add Archive option

---
 distribution.c  | 28 +++++++++++++++
 distribution.h  |  2 ++
 docs/reprepro.1 | 14 ++++++--
 target.c        | 93 ++++++++++++++++++++++++++++++++++++++++++++++---
 4 files changed, 129 insertions(+), 8 deletions(-)

diff --git a/distribution.c b/distribution.c
index d56a8dd8..bfa1ff54 100644
--- a/distribution.c
+++ b/distribution.c
@@ -47,6 +47,7 @@ static retvalue distribution_free(struct distribution *distribution) {
 	bool needsretrack = false;
 
 	if (distribution != NULL) {
+		distribution->archive = NULL;
 		free(distribution->suite);
 		free(distribution->fakecomponentprefix);
 		free(distribution->version);
@@ -423,6 +424,32 @@ CFcheckvalueSETPROC(distribution, codename, checkforcodename)
 CFcheckvalueSETPROC(distribution, fakecomponentprefix, checkfordirectoryandidentifier)
 CFtimespanSETPROC(distribution, validfor)
 
+CFuSETPROC(distribution, archive) {
+	CFSETPROCVARS(distribution, data, mydata);
+	char *codename;
+	retvalue r;
+
+	r = config_getall(iter, &codename);
+	if (!RET_IS_OK(r))
+		return r;
+
+	for (struct distribution *d = mydata->distributions; d != NULL; d = d->next) {
+		if (strcmp(d->codename, codename) == 0) {
+			data->archive = d;
+			free(codename);
+			return RET_OK;
+		}
+	}
+
+	fprintf(stderr,
+"Error parsing config file %s, line %u:\n"
+"No distribution has '%s' as codename.\n"
+"Note: The archive distribution '%s' must be specified before '%s'.\n",
+		config_filename(iter), config_line(iter), codename, codename, data->codename);
+	free(codename);
+	return RET_ERROR_MISSING;
+}
+
 CFUSETPROC(distribution, Contents) {
 	CFSETPROCVAR(distribution, d);
 	return contentsoptions_parse(d, iter);
@@ -458,6 +485,7 @@ CFUSETPROC(distribution, exportoptions) {
 static const struct configfield distributionconfigfields[] = {
 	CF("AlsoAcceptFor",	distribution,	alsoaccept),
 	CFr("Architectures",	distribution,	architectures),
+	CF("Archive",		distribution,   archive),
 	CF("ByHandHooks",	distribution,	byhandhooks),
 	CFr("Codename",		distribution,	codename),
 	CFr("Components",	distribution,	components),
diff --git a/distribution.h b/distribution.h
index bc658b5a..c396f44e 100644
--- a/distribution.h
+++ b/distribution.h
@@ -53,6 +53,8 @@ struct distribution {
 	/* the key to sign with, may have no entries to mean unsigned: */
 	struct strlist signwith;
 	long long limit;
+	/* the codename of the archive distribution (when the limit is exceeded) */
+	/*@null@*/struct distribution *archive;
 	/* the override file to use by default */
 	/*@null@*/char *deb_override, *udeb_override, *dsc_override;
 	/* fake component prefix (and codename antisuffix) for Release files: */
diff --git a/docs/reprepro.1 b/docs/reprepro.1
index ac739782..318181ac 100644
--- a/docs/reprepro.1
+++ b/docs/reprepro.1
@@ -1603,9 +1603,17 @@ Not yet implemented.
 .B Limit
 Limit the number of versions of a package per distribution, architecture,
 component, and type. The limit must be a number. If the number is positive,
-all old package version that exceed these limit will be removed when a new
-package version is added. If the number is zero or negative, all package
-version will be kept. By default only one package version will be kept.
+all old package version that exceed these limit will be removed or archived
+(see
+.B Archive
+option), when a new package version is added. If the number is zero or negative,
+all package version will be kept. By default only one package version will be
+kept.
+.TP
+.B Archive
+Specify a codename which must be declared before (to avoid loops). When packages
+exceed the version count limit (specified in \fBLimit\fR), these packages will
+be moved to the specified distribution instead of being removed.
 .TP
 .B Log
 Specify a file to log additions and removals of this distribution
diff --git a/target.c b/target.c
index 67d64787..c458ae79 100644
--- a/target.c
+++ b/target.c
@@ -346,6 +346,80 @@ retvalue package_remove_by_cursor(struct package_cursor *tc, struct logger *logg
 	return result;
 }
 
+static retvalue archive_package(struct target *target, const struct package *package, const struct strlist *files, /*@null@*/const char *causingrule, /*@null@*/const char *suitefrom) {
+	struct strlist filekeys;
+	struct target *archive_target;
+	struct trackingdata trackingdata;
+	trackingdb tracks = NULL;
+	bool close_database, close_trackingdb = false;
+	retvalue result, r;
+
+	if (target->distribution->archive != NULL) {
+		archive_target = distribution_gettarget(target->distribution->archive, target->component,
+		                                        target->architecture, target->packagetype);
+		if (archive_target == NULL) {
+			fprintf(stderr,
+"Warning: Cannot archive '%s=%s' from '%s' to '%s' since '%s' has no matching component/architecture/packagetype.\n",
+			        package->name, package->version, target->distribution->codename,
+			        target->distribution->archive->codename,
+			        target->distribution->archive->codename);
+		} else {
+			close_database = archive_target->packages == NULL;
+			if (close_database) {
+				result = target_initpackagesdb(archive_target, READWRITE);
+				if (RET_WAS_ERROR(result)) {
+					return result;
+				}
+			}
+			if (files == NULL) {
+				result = archive_target->getfilekeys(package->control, &filekeys);
+				if (RET_WAS_ERROR(result))
+					return result;
+				files = &filekeys;
+			}
+			if (archive_target->distribution->tracking != dt_NONE) {
+				close_trackingdb = archive_target->distribution->trackingdb == NULL;
+				if (close_trackingdb) {
+					r = tracking_initialize(&tracks, archive_target->distribution, false);
+					if (RET_WAS_ERROR(r))
+						return r;
+				} else {
+					tracks = archive_target->distribution->trackingdb;
+				}
+				r = trackingdata_summon(tracks, package->source, package->version, &trackingdata);
+				if (RET_WAS_ERROR(r))
+					return r;
+			}
+			// TODO: Check whether this is the best place to set 'selected'
+			target->distribution->archive->selected = true;
+			result = distribution_prepareforwriting(archive_target->distribution);
+			if (!RET_WAS_ERROR(result)) {
+				result = target_addpackage(archive_target, target->distribution->archive->logger,
+					              package->name, package->version, package->control,
+					              files, false, (tracks != NULL) ? &trackingdata : NULL,
+					              target->architecture, causingrule, suitefrom);
+				RET_UPDATE(target->distribution->archive->status, result);
+			}
+			if (close_database) {
+				r = target_closepackagesdb(archive_target);
+				RET_UPDATE(result, r);
+			}
+			if (tracks != NULL) {
+				r = trackingdata_finish(tracks, &trackingdata);
+				RET_UPDATE(result, r);
+				if (close_trackingdb) {
+					r = tracking_done(tracks, archive_target->distribution);
+					RET_UPDATE(result, r);
+				}
+			}
+			if (RET_WAS_ERROR(result)) {
+				return result;
+			}
+		}
+	}
+	return RET_OK;
+}
+
 static retvalue addpackages(struct target *target, const char *packagename, const char *controlchunk, const char *version, const struct strlist *files, /*@null@*/const struct package *old, /*@null@*/const struct strlist *oldfiles, /*@null@*/struct logger *logger, /*@null@*/struct trackingdata *trackingdata, architecture_t architecture, /*@null@*/const char *causingrule, /*@null@*/const char *suitefrom) {
 
 	retvalue result = RET_OK, r;
@@ -373,11 +447,13 @@ static retvalue addpackages(struct target *target, const char *packagename, cons
 
 	if (old != NULL && old->control != NULL) {
 		key = package_primarykey(old->name, old->version);
-		r = table_deleterecord(table, key, false);
-		free(key);
-		if (RET_WAS_ERROR(r)) {
-			return r;
+		r = archive_package(target, old, oldfiles, causingrule, suitefrom);
+		RET_UPDATE(result, r);
+		if (RET_IS_OK(r)) {
+			r = table_deleterecord(table, key, false);
+			RET_UPDATE(result, r);
 		}
+		free(key);
 	}
 
 	key = package_primarykey(packagename, version);
@@ -565,9 +641,16 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 			if (RET_WAS_ERROR(r2))
 				continue;
 			if (strcmp(version, iterator.current.version) == 0) {
-				// Do not remove the newly added package!
+				// Do not archive/remove the newly added package!
 				continue;
 			}
+			r2 = package_getsource(&iterator.current);
+			if (RET_WAS_ERROR(r2))
+				continue;
+			r2 = archive_package(target, &iterator.current, NULL, causingrule, suitefrom);
+			RET_UPDATE(r, r2);
+			if (RET_WAS_ERROR(r2))
+				continue;
 			r2 = package_remove_by_cursor(&iterator, logger, trackingdata);
 			RET_UPDATE(r, r2);
 		}

From b6a33e20a509f6b3c7eeaf4222d946bbccf93d2f Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 28 Feb 2017 18:13:40 +0100
Subject: [PATCH 64/70] [testsuite] Add test cases for Archive option

---
 tests/basic.sh        | 173 ++++++++++++++++++++++++++++++++++++++++++
 tests/multiversion.sh |  36 +++++++++
 2 files changed, 209 insertions(+)

diff --git a/tests/basic.sh b/tests/basic.sh
index 4cd5f187..f4a0f39c 100755
--- a/tests/basic.sh
+++ b/tests/basic.sh
@@ -239,4 +239,177 @@ buster|main|$ARCH: sl 3.03-1
 buster|main|source: sl 3.03-1" "$($REPREPRO -b $REPO list buster)"
 }
 
+test_archive() {
+	clear_distro
+	add_distro buster-archive
+	add_distro buster "Limit: 1\nArchive: buster-archive"
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-1_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-2_${ARCH}.deb
+	assertEquals "\
+hello | 2.9-1 | buster-archive | $ARCH
+hello | 2.9-2 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster-archive
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1
+
+Distribution: buster
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+}
+
+test_archive_downgrade() {
+	clear_distro
+	add_distro buster-archive
+	add_distro buster "Limit: 1\nArchive: buster-archive"
+	add_distro buster-proposed
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-2_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster-proposed $PKGS/hello_2.9-1_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO move buster buster-proposed hello=2.9-1
+	assertEquals "\
+hello | 2.9-2 | buster-archive | $ARCH
+hello | 2.9-1 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster-archive
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1
+
+Distribution: buster
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+}
+
+test_archive_move() {
+	clear_distro
+	add_distro buster-archive "Limit: -1"
+	add_distro buster "Limit: 1\nArchive: buster-archive"
+	add_distro buster-proposed "Limit: -1"
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-3 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster-proposed $PKGS/hello_2.9-1_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster-proposed $PKGS/hello_2.9-2_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster-proposed $PKGS/hello_2.9-3_${ARCH}.deb
+	assertEquals "\
+hello | 2.9-3 | buster-proposed | $ARCH
+hello | 2.9-2 | buster-proposed | $ARCH
+hello | 2.9-1 | buster-proposed | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster-proposed
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1
+
+Distribution: buster-proposed
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1
+
+Distribution: buster-proposed
+Source: hello
+Version: 2.9-3
+Files:
+ pool/main/h/hello/hello_2.9-3_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main move buster buster-proposed hello=2.9-1
+	assertEquals "\
+hello | 2.9-1 |          buster | $ARCH
+hello | 2.9-3 | buster-proposed | $ARCH
+hello | 2.9-2 | buster-proposed | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1
+
+Distribution: buster-proposed
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1
+
+Distribution: buster-proposed
+Source: hello
+Version: 2.9-3
+Files:
+ pool/main/h/hello/hello_2.9-3_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main move buster buster-proposed hello=2.9-2
+	assertEquals "\
+hello | 2.9-1 |  buster-archive | $ARCH
+hello | 2.9-2 |          buster | $ARCH
+hello | 2.9-3 | buster-proposed | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster-archive
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1
+
+Distribution: buster
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1
+
+Distribution: buster-proposed
+Source: hello
+Version: 2.9-3
+Files:
+ pool/main/h/hello/hello_2.9-3_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main move buster buster-proposed hello
+	assertEquals "\
+hello | 2.9-2 | buster-archive | $ARCH
+hello | 2.9-1 | buster-archive | $ARCH
+hello | 2.9-3 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster-archive
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1
+
+Distribution: buster-archive
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1
+
+Distribution: buster
+Source: hello
+Version: 2.9-3
+Files:
+ pool/main/h/hello/hello_2.9-3_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+}
+
+test_archive_move_back() {
+	clear_distro
+	add_distro buster-archive "Limit: -1"
+	add_distro buster "Limit: 1\nArchive: buster-archive"
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 ../genpackage.sh)
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-2 ../genpackage.sh)
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-1_${ARCH}.deb
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster-archive $PKGS/hello_2.9-2_${ARCH}.deb
+	assertEquals "\
+hello | 2.9-2 | buster-archive | $ARCH
+hello | 2.9-1 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main move buster buster-archive hello=2.9-2
+	assertEquals "\
+hello | 2.9-1 | buster-archive | $ARCH
+hello | 2.9-2 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+}
+
 . shunit2
diff --git a/tests/multiversion.sh b/tests/multiversion.sh
index 4562a9d3..a7b83ee1 100755
--- a/tests/multiversion.sh
+++ b/tests/multiversion.sh
@@ -196,6 +196,42 @@ Files:
  pool/main/h/hello/hello_2.9-10_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
 }
 
+test_reduce_limit_archive() {
+	clear_distro
+	add_distro buster-archive "Limit: 7"
+	add_distro buster "Limit: -1\nArchive: buster-archive"
+	for revision in 1 2; do
+		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb
+	done
+	assertEquals "\
+buster|main|${ARCH}: hello 2.9-2
+buster|main|${ARCH}: hello 2.9-1" "$($REPREPRO -b $REPO list buster)"
+	sed -i 's/^Limit: -1$/Limit: 1/' $REPO/conf/distributions
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-10_${ARCH}.deb
+	assertEquals "\
+hello |  2.9-2 | buster-archive | $ARCH
+hello |  2.9-1 | buster-archive | $ARCH
+hello | 2.9-10 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
+	assertEquals "\
+Distribution: buster-archive
+Source: hello
+Version: 2.9-1
+Files:
+ pool/main/h/hello/hello_2.9-1_$ARCH.deb b 1
+
+Distribution: buster-archive
+Source: hello
+Version: 2.9-2
+Files:
+ pool/main/h/hello/hello_2.9-2_$ARCH.deb b 1
+
+Distribution: buster
+Source: hello
+Version: 2.9-10
+Files:
+ pool/main/h/hello/hello_2.9-10_$ARCH.deb b 1" "$($REPREPRO -b $REPO dumptracks)"
+}
+
 test_limit_old() {
 	for revision in 1 2 10; do
 		call $REPREPRO $VERBOSE_ARGS -b $REPO -C main includedeb buster $PKGS/hello_2.9-${revision}_${ARCH}.deb

From a74882038f861b0f3813ef3923322fc243056169 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 12 Apr 2017 14:17:33 +0200
Subject: [PATCH 65/70] Accept .ddeb files as dbgsym packages

Closes: #730572
LP: #799889
---
 atoms.c             | 12 +++++++---
 atoms.h             |  1 +
 changes.c           |  6 +++--
 changes.h           |  6 ++---
 checkin.c           | 23 ++++++++++++++++++++
 checkindeb.c        | 53 +++++++++++++++++++++++++++++++++++++--------
 contents.c          | 48 +++++++++++++++++++++++++++++++++++-----
 contents.h          |  1 +
 copypackages.c      |  9 +++++++-
 distribution.c      | 45 ++++++++++++++++++++++++++++++++++++++
 distribution.h      |  7 +++++-
 docs/reprepro.1     |  3 +++
 incoming.c          | 15 +++++++++++++
 main.c              | 25 ++++++++++++++++++---
 pull.c              |  2 ++
 target.c            | 23 ++++++++++++++++++++
 target.h            |  1 +
 tests/basic.sh      | 10 +++++++++
 tests/genpackage.sh | 31 ++++++++++++++++++++++++--
 tool.c              | 18 ++++++++++-----
 updates.c           |  3 +++
 21 files changed, 307 insertions(+), 35 deletions(-)

diff --git a/atoms.c b/atoms.c
index 62f8b09e..a8309fd3 100644
--- a/atoms.c
+++ b/atoms.c
@@ -26,7 +26,7 @@
 
 const char **atoms_architectures;
 const char **atoms_components;
-const char * const packagetypes[4] = { "!!NONE!!", "dsc", "deb", "udeb" };
+const char * const packagetypes[5] = { "!!NONE!!", "dsc", "deb", "udeb", "ddeb" };
 const char **atoms_packagetypes = (const char **)&packagetypes;
 const char **atoms_commands;
 static int command_count;
@@ -159,6 +159,8 @@ packagetype_t packagetype_find(const char *value) {
 		return pt_deb;
 	else if (strcmp(value, "udeb") == 0)
 		return pt_udeb;
+	else if (strcmp(value, "ddeb") == 0)
+		return pt_ddeb;
 	else
 		return atom_unknown;
 }
@@ -169,8 +171,12 @@ packagetype_t packagetype_find_l(const char *value, size_t len) {
 			return pt_dsc;
 		else if (strncmp(value, "deb", 3) == 0)
 			return pt_deb;
-	} else if (len == 4 && strncmp(value, "udeb", 4) == 0)
-		return pt_udeb;
+	} else if (len == 4) {
+		if (strncmp(value, "udeb", 4) == 0)
+			return pt_udeb;
+		else if (strncmp(value, "ddeb", 4) == 0)
+			return pt_ddeb;
+	}
 	return atom_unknown;
 }
 
diff --git a/atoms.h b/atoms.h
index be787139..660adb43 100644
--- a/atoms.h
+++ b/atoms.h
@@ -19,6 +19,7 @@ enum atom_type { at_architecture, at_component, at_packagetype, at_command };
 #define pt_dsc ((packagetype_t)1)
 #define pt_deb ((packagetype_t)2)
 #define pt_udeb ((packagetype_t)3)
+#define pt_ddeb ((packagetype_t)4)
 
 #define atom_defined(a) ((a) > (atom_t)0)
 
diff --git a/changes.c b/changes.c
index 2cfb33d4..8b4b5c82 100644
--- a/changes.c
+++ b/changes.c
@@ -164,6 +164,8 @@ retvalue changes_parsefileline(const char *fileline, /*@out@*/filetype *result_t
 
 	if (l >= 4 && memcmp(p-4, ".deb", 4) == 0)
 		type = fe_DEB;
+	else if (l >= 5 && memcmp(p-5, ".ddeb", 5) == 0)
+		type = fe_DDEB;
 	else if (l >= 5 && memcmp(p-5, ".udeb", 5) == 0)
 		type = fe_UDEB;
 	else
@@ -189,7 +191,7 @@ retvalue changes_parsefileline(const char *fileline, /*@out@*/filetype *result_t
 		if (type == fe_DEB)
 			archend = versionstart + l - 4;
 		else {
-			assert (type == fe_UDEB);
+			assert (type == fe_DDEB || type == fe_UDEB);
 			archend = versionstart + l - 5;
 		}
 		if (archend - archstart == 6 &&
@@ -204,7 +206,7 @@ retvalue changes_parsefileline(const char *fileline, /*@out@*/filetype *result_t
 		bool issignature = false;
 
 		/* without those, it gets more complicated.
-		 * It's not .deb or .udeb, so most likely a
+		 * It's not .deb, .ddeb or .udeb, so most likely a
 		 * source file (or perhaps a log (reprepro extension)) */
 
 		/* if it uses a known compression, things are easy,
diff --git a/changes.h b/changes.h
index b4206d9b..d046e18c 100644
--- a/changes.h
+++ b/changes.h
@@ -7,7 +7,7 @@
 
 typedef enum {
 	fe_UNKNOWN=0,
-	fe_DEB, fe_UDEB,
+	fe_DEB, fe_UDEB, fe_DDEB,
 	fe_DSC, fe_DIFF, fe_ORIG, fe_TAR,
 	fe_SIG,
 	fe_ALTSRC,
@@ -15,8 +15,8 @@ typedef enum {
 	fe_BUILDINFO
 } filetype;
 
-#define FE_PACKAGE(ft) ((ft) == fe_DEB || (ft) == fe_UDEB || (ft) == fe_DSC)
-#define FE_BINARY(ft) ((ft) == fe_DEB || (ft) == fe_UDEB)
+#define FE_PACKAGE(ft) ((ft) == fe_DEB || (ft) == fe_UDEB || (ft) == fe_DSC || (ft) == fe_DDEB)
+#define FE_BINARY(ft) ((ft) == fe_DEB || (ft) == fe_DDEB || (ft) == fe_UDEB)
 #define FE_SOURCE(ft) ((ft) == fe_DIFF || (ft) == fe_ORIG || (ft) == fe_TAR || (ft) == fe_DSC || (ft) == fe_UNKNOWN || (ft) == fe_ALTSRC || (ft) == fe_SIG)
 
 struct hash_data;
diff --git a/checkin.c b/checkin.c
index 8bedffa2..8ff7fe37 100644
--- a/checkin.c
+++ b/checkin.c
@@ -207,6 +207,11 @@ static retvalue newentry(struct fileentry **entry, const char *fileline, const s
 		*ignoredlines_p = true;
 		return RET_NOTHING;
 	}
+	if (e->type == fe_DDEB && limitations_missed(packagetypes, pt_ddeb)) {
+		freeentries(e);
+		*ignoredlines_p = true;
+		return RET_NOTHING;
+	}
 	if (e->type != fe_LOG && e->type != fe_BUILDINFO &&
 			e->architecture_into == architecture_source &&
 			strcmp(e->name, sourcename) != 0) {
@@ -1243,6 +1248,15 @@ static retvalue changes_checkpkgs(struct distribution *distribution, struct chan
 				e->filekey, e->checksums,
 				&changes->binaries,
 				changes->source, changes->sourceversion);
+		} else if (e->type == fe_DDEB) {
+			r = deb_prepare(&e->pkg.deb,
+				e->component, e->architecture_into,
+				e->section, e->priority,
+				pt_ddeb,
+				distribution, fullfilename,
+				e->filekey, e->checksums,
+				&changes->binaries,
+				changes->source, changes->sourceversion);
 		} else if (e->type == fe_DSC) {
 			if (!changes->isbinnmu || IGNORING(dscinbinnmu,
 "File '%s' looks like a source package, but this .changes looks like a binNMU\n"
@@ -1291,6 +1305,15 @@ static retvalue changes_includepkgs(struct distribution *distribution, struct ch
 				pt_deb, distribution, trackingdata);
 			if (r == RET_NOTHING)
 				*missed_p = true;
+		} else if (e->type == fe_DDEB) {
+			r = deb_addprepared(e->pkg.deb,
+				/* architecture all needs this, the rest is
+				 * already filtered out */
+				(e->architecture_into == architecture_all)?
+					forcearchitectures:NULL,
+				pt_ddeb, distribution, trackingdata);
+			if (r == RET_NOTHING)
+				*missed_p = true;
 		} else if (e->type == fe_UDEB) {
 			r = deb_addprepared(e->pkg.deb,
 				/* architecture all needs this, the rest is
diff --git a/checkindeb.c b/checkindeb.c
index 6d03c00a..1b89cefd 100644
--- a/checkindeb.c
+++ b/checkindeb.c
@@ -105,7 +105,13 @@ static retvalue deb_preparelocation(struct debpackage *pkg, component_t forcecom
 	const struct overridedata *oinfo;
 	retvalue r;
 
-	if (packagetype == pt_udeb) {
+	if (packagetype == pt_ddeb) {
+		/* ddebs don't have overrides */
+		forcesection = "debug";
+		forcepriority = "extra";
+		binoverride = NULL;
+		components = &distribution->ddebcomponents;
+	} else if (packagetype == pt_udeb) {
 		binoverride = distribution->overrides.udeb;
 		components = &distribution->udebcomponents;
 	} else {
@@ -113,14 +119,21 @@ static retvalue deb_preparelocation(struct debpackage *pkg, component_t forcecom
 		components = &distribution->components;
 	}
 
-	oinfo = override_search(binoverride, pkg->deb.name);
-	*oinfo_ptr = oinfo;
-	if (forcesection == NULL) {
-		forcesection = override_get(oinfo, SECTION_FIELDNAME);
-	}
-	if (forcepriority == NULL) {
-		forcepriority = override_get(oinfo, PRIORITY_FIELDNAME);
+	if (binoverride == NULL) {
+		oinfo = NULL;
+	} else {
+		oinfo = override_search(binoverride, pkg->deb.name);
+
+		if (forcesection == NULL) {
+			forcesection = override_get(oinfo, SECTION_FIELDNAME);
+		}
+
+		if (forcepriority == NULL) {
+			forcepriority = override_get(oinfo, PRIORITY_FIELDNAME);
+		}
 	}
+	*oinfo_ptr = oinfo;
+
 	if (!atom_defined(forcecomponent)) {
 		const char *fc;
 
@@ -261,7 +274,29 @@ retvalue deb_prepare(/*@out@*/struct debpackage **deb, component_t forcecomponen
 		base = NULL;
 	}
 
-	if (!strlist_in(allowed_binaries, packagenametocheck) &&
+	if (packagetype == pt_ddeb) {
+		/* ddebs are allowed if they are an allowed
+		 * binary + "-dbgsym" */
+		int i;
+		bool found = false;
+
+		for (i = 0; i < allowed_binaries->count; i++) {
+			const char *s = allowed_binaries->values[i];
+			size_t len = strlen(s);
+
+			if (strncmp(s, pkg->deb.name, len) == 0 &&
+					strcmp(pkg->deb.name + len, "-dbgsym") == 0) {
+				found = true;
+			}
+		}
+
+		if (!found && !IGNORING(surprisingbinary,
+					"'%s' has packagename '%s' not corresponding to a .deb listed in the .changes file!\n",
+					debfilename, pkg->deb.name)) {
+			deb_free(pkg);
+			return RET_ERROR;
+		}
+	} else if (!strlist_in(allowed_binaries, packagenametocheck) &&
 	    !IGNORING(surprisingbinary,
 "'%s' has packagename '%s' not listed in the .changes file!\n",
 					debfilename, packagenametocheck)) {
diff --git a/contents.c b/contents.c
index 40da6bc8..22ef046f 100644
--- a/contents.c
+++ b/contents.c
@@ -43,6 +43,7 @@ retvalue contentsoptions_parse(struct distribution *distribution, struct configi
 		cf_uncompressed, cf_gz, cf_bz2, cf_xz,
 		cf_percomponent, cf_allcomponents,
 		cf_compatsymlink, cf_nocompatsymlink,
+		cf_ddebs,
 		cf_COUNT
 	};
 	bool flags[cf_COUNT];
@@ -60,6 +61,7 @@ retvalue contentsoptions_parse(struct distribution *distribution, struct configi
 		{".bz2", cf_bz2},
 		{".gz", cf_gz},
 		{".", cf_uncompressed},
+		{"ddebs", cf_ddebs},
 		{NULL, -1}
 	};
 	retvalue r;
@@ -130,6 +132,7 @@ retvalue contentsoptions_parse(struct distribution *distribution, struct configi
 		distribution->contents.compressions |= IC_FLAG(ic_xz);
 #endif
 	distribution->contents.flags.udebs = flags[cf_udebs];
+	distribution->contents.flags.ddebs = flags[cf_ddebs];
 	distribution->contents.flags.nodebs = flags[cf_nodebs];
 	if (flags[cf_allcomponents])
 		distribution->contents.flags.allcomponents = true;
@@ -173,20 +176,36 @@ static retvalue gentargetcontents(struct target *target, struct release *release
 	struct filetorelease *file;
 	struct filelist_list *contents;
 	struct package_cursor iterator;
+	const char *suffix;
+	const char *symlink_prefix;
 
 	if (onlyneeded && target->saved_wasmodified)
 		onlyneeded = false;
 
+	switch (target->packagetype) {
+		case pt_ddeb:
+			symlink_prefix = "d";
+			suffix = "-ddeb";
+			break;
+		case pt_udeb:
+			symlink_prefix = "s";
+			suffix = "-udeb";
+			break;
+		default:
+			symlink_prefix = "";
+			suffix = "";
+	}
+
 	contentsfilename = mprintf("%s/Contents%s-%s",
 			atoms_components[target->component],
-			(target->packagetype == pt_udeb)?"-udeb":"",
+			suffix,
 			atoms_architectures[target->architecture]);
 	if (FAILEDTOALLOC(contentsfilename))
 		return RET_ERROR_OOM;
 
 	if (symlink) {
 		char *symlinkas = mprintf("%sContents-%s",
-				(target->packagetype == pt_udeb)?"s":"",
+				symlink_prefix,
 				atoms_architectures[target->architecture]);
 		if (FAILEDTOALLOC(symlinkas)) {
 			free(contentsfilename);
@@ -244,17 +263,30 @@ static retvalue genarchcontents(struct distribution *distribution, architecture_
 	const struct atomlist *components;
 	struct target *target;
 	bool combinedonlyifneeded;
+	const char *prefix;
+	const char *symlink_prefix;
 
-	if (type == pt_udeb) {
+	if (type == pt_ddeb) {
+		if (distribution->contents_components_set)
+			components = &distribution->contents_dcomponents;
+		else
+			components = &distribution->ddebcomponents;
+		prefix = "d";
+		symlink_prefix = "d";
+	} else if (type == pt_udeb) {
 		if (distribution->contents_components_set)
 			components = &distribution->contents_ucomponents;
 		else
 			components = &distribution->udebcomponents;
+		prefix = "u";
+		symlink_prefix = "s";
 	} else {
 		if (distribution->contents_components_set)
 			components = &distribution->contents_components;
 		else
 			components = &distribution->components;
+		prefix = "";
+		symlink_prefix = "";
 	}
 
 	if (components->count == 0)
@@ -286,7 +318,7 @@ static retvalue genarchcontents(struct distribution *distribution, architecture_
 	if (!distribution->contents.flags.allcomponents) {
 		if (!distribution->contents.flags.compatsymlink) {
 			char *symlinkas = mprintf("%sContents-%s",
-					(type == pt_udeb)?"s":"",
+					symlink_prefix,
 					atoms_architectures[architecture]);
 			if (FAILEDTOALLOC(symlinkas))
 				return RET_ERROR_OOM;
@@ -298,7 +330,7 @@ static retvalue genarchcontents(struct distribution *distribution, architecture_
 	}
 
 	contentsfilename = mprintf("%sContents-%s",
-			(type == pt_udeb)?"u":"",
+			prefix,
 			atoms_architectures[architecture]);
 	if (FAILEDTOALLOC(contentsfilename))
 		return RET_ERROR_OOM;
@@ -365,6 +397,12 @@ retvalue contents_generate(struct distribution *distribution, struct release *re
 					release, onlyneeded);
 			RET_UPDATE(result, r);
 		}
+		if (distribution->contents.flags.ddebs) {
+			r = genarchcontents(distribution,
+					architecture, pt_ddeb,
+					release, onlyneeded);
+			RET_UPDATE(result, r);
+		}
 	}
 	return result;
 }
diff --git a/contents.h b/contents.h
index 921f6a00..b8a214d2 100644
--- a/contents.h
+++ b/contents.h
@@ -19,6 +19,7 @@ struct contentsoptions {
 		bool percomponent;
 		bool allcomponents;
 		bool compatsymlink;
+		bool ddebs;
 	} flags;
 	compressionset compressions;
 };
diff --git a/copypackages.c b/copypackages.c
index ae2f4f94..317627a2 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -846,7 +846,14 @@ retvalue copy_from_file(struct distribution *into, component_t component, archit
 					into->codename,
 					atoms_architectures[architecture]);
 		}
-		if (packagetype != pt_udeb) {
+		if (packagetype == pt_ddeb) {
+			if (!atomlist_in(&into->ddebcomponents, component)) {
+				fprintf(stderr,
+"Distribution '%s' does not contain ddeb component '%s!'\n",
+					into->codename,
+					atoms_components[component]);
+			}
+		} else if (packagetype != pt_udeb) {
 			if (!atomlist_in(&into->components, component)) {
 				fprintf(stderr,
 "Distribution '%s' does not contain component '%s!'\n",
diff --git a/distribution.c b/distribution.c
index bfa1ff54..23c63b6c 100644
--- a/distribution.c
+++ b/distribution.c
@@ -71,8 +71,10 @@ static retvalue distribution_free(struct distribution *distribution) {
 		exportmode_done(&distribution->dsc);
 		exportmode_done(&distribution->deb);
 		exportmode_done(&distribution->udeb);
+		exportmode_done(&distribution->ddeb);
 		atomlist_done(&distribution->contents_architectures);
 		atomlist_done(&distribution->contents_components);
+		atomlist_done(&distribution->contents_dcomponents);
 		atomlist_done(&distribution->contents_ucomponents);
 		override_free(distribution->overrides.deb);
 		override_free(distribution->overrides.udeb);
@@ -187,6 +189,26 @@ static retvalue createtargets(struct distribution *distribution) {
 					return r;
 
 			}
+			if (atomlist_in(&distribution->ddebcomponents, c)) {
+				r = target_initialize_dbinary(
+						distribution,
+						c, a,
+						&distribution->ddeb,
+						distribution->readonly,
+						distribution->exportoptions[deo_noexport],
+						distribution->fakecomponentprefix,
+						&t);
+				if (RET_IS_OK(r)) {
+					if (last != NULL) {
+						last->next = t;
+					} else {
+						distribution->targets = t;
+					}
+					last = t;
+				}
+				if (RET_WAS_ERROR(r))
+					return r;
+			}
 		}
 		/* check if this distribution contains source
 		 * (yes, yes, source is not really an architecture, but
@@ -230,6 +252,11 @@ CFstartparse(distribution) {
 		(void)distribution_free(n);
 		return r;
 	}
+	r = exportmode_init(&n->ddeb, true, "Release", "Packages");
+	if (RET_WAS_ERROR(r)) {
+		(void)distribution_free(n);
+		return r;
+	}
 	r = exportmode_init(&n->deb, true, "Release", "Packages");
 	if (RET_WAS_ERROR(r)) {
 		(void)distribution_free(n);
@@ -332,6 +359,9 @@ CFfinishparse(distribution) {
 	    notpropersuperset(&n->components, "Components",
 			    &n->contents_components, "ContentsComponents",
 			    atoms_components, n) ||
+	    notpropersuperset(&n->ddebcomponents, "DDebComponents",
+			    &n->contents_dcomponents, "ContentsDComponents",
+			    atoms_components, n) ||
 	    notpropersuperset(&n->udebcomponents, "UDebComponents",
 			    &n->contents_ucomponents, "ContentsUComponents",
 			    atoms_components, n) ||
@@ -339,6 +369,9 @@ CFfinishparse(distribution) {
 	    // in the rest of the code...:
 	    notpropersuperset(&n->components, "Components",
 			    &n->udebcomponents, "UDebComponents",
+			    atoms_components, n) ||
+	    notpropersuperset(&n->components, "Components",
+			    &n->ddebcomponents, "DDebComponents",
 			    atoms_components, n)) {
 		(void)distribution_free(n);
 		return RET_ERROR;
@@ -360,6 +393,14 @@ CFfinishparse(distribution) {
 			n->contents.flags.udebs = false;
 		}
 	}
+	if (n->contents_dcomponents_set) {
+		if (n->contents_dcomponents.count > 0) {
+			n->contents.flags.enabled = true;
+			n->contents.flags.ddebs = true;
+		} else {
+			n->contents.flags.ddebs = false;
+		}
+	}
 	if (n->contents_architectures_set) {
 		if (n->contents_architectures.count > 0)
 			n->contents.flags.enabled = true;
@@ -415,8 +456,10 @@ CFinternatomsSETPROC(distribution, components, checkforcomponent, at_component)
 CFinternatomsSETPROC(distribution, architectures, checkforarchitecture, at_architecture)
 CFatomsublistSETPROC(distribution, contents_architectures, at_architecture, architectures, "Architectures")
 CFatomsublistSETPROC(distribution, contents_components, at_component, components, "Components")
+CFatomsublistSETPROC(distribution, ddebcomponents, at_component, components, "Components")
 CFatomsublistSETPROC(distribution, udebcomponents, at_component, components, "Components")
 CFatomsublistSETPROC(distribution, contents_ucomponents, at_component, udebcomponents, "UDebComponents")
+CFexportmodeSETPROC(distribution, ddeb)
 CFexportmodeSETPROC(distribution, udeb)
 CFexportmodeSETPROC(distribution, deb)
 CFexportmodeSETPROC(distribution, dsc)
@@ -493,6 +536,8 @@ static const struct configfield distributionconfigfields[] = {
 	CF("ContentsComponents", distribution,	contents_components),
 	CF("Contents",		distribution,	Contents),
 	CF("ContentsUComponents", distribution,	contents_ucomponents),
+	CF("DDebComponents",	distribution,	ddebcomponents),
+	CF("DDebIndices",	distribution,	ddeb),
 	CF("DebIndices",	distribution,	deb),
 	CF("DebOverride",	distribution,	deb_override),
 	CF("Description",	distribution,	description),
diff --git a/distribution.h b/distribution.h
index c396f44e..10780844 100644
--- a/distribution.h
+++ b/distribution.h
@@ -66,8 +66,10 @@ struct distribution {
 	/* the list of components containing a debian-installer dir,
 	 * normally only "main" */
 	struct atomlist udebcomponents;
+	/* the list of components containing a debug directory */
+	struct atomlist ddebcomponents;
 	/* what kind of index files to generate */
-	struct exportmode dsc, deb, udeb;
+	struct exportmode dsc, deb, udeb, ddeb;
 	bool exportoptions[deo_COUNT];
 	/* (NONE must be 0 so it is the default) */
 	enum trackingtype { dt_NONE=0, dt_KEEP, dt_ALL, dt_MINIMAL } tracking;
@@ -84,11 +86,14 @@ struct distribution {
 	struct contentsoptions contents;
 	struct atomlist contents_architectures,
 		       contents_components,
+		       contents_dcomponents,
 		       contents_ucomponents;
 	bool contents_architectures_set,
 		       contents_components_set,
+		       contents_dcomponents_set,
 		       contents_ucomponents_set,
 		       /* not used, just here to keep things simpler: */
+		       ddebcomponents_set,
 		       udebcomponents_set;
 	/* A list of all targets contained in the distribution*/
 	struct target *targets;
diff --git a/docs/reprepro.1 b/docs/reprepro.1
index 318181ac..a3c57cfa 100644
--- a/docs/reprepro.1
+++ b/docs/reprepro.1
@@ -1380,6 +1380,9 @@ distribution. See
 for rules which component packages are included into
 by default. This will also be copied into the Release files.
 .TP
+.B DDebComponents
+List of components containing .ddebs.
+.TP
 .B UDebComponents
 Components with a debian\-installer subhierarchy containing .udebs.
 (E.g. simply "main")
diff --git a/incoming.c b/incoming.c
index 60361dae..2ffa9651 100644
--- a/incoming.c
+++ b/incoming.c
@@ -1267,9 +1267,13 @@ static retvalue prepare_deb(const struct incoming *i, const struct candidate *c,
 	assert (file == package->master);
 	if (file->type == fe_DEB)
 		package->packagetype = pt_deb;
+	else if (file->type == fe_DDEB)
+		package->packagetype = pt_ddeb;
 	else
 		package->packagetype = pt_udeb;
 
+	/* we use the deb overrides for ddebs too - ddebs aren't
+	 * meant to have overrides so this is probably fine */
 	oinfo = override_search(file->type==fe_UDEB?into->overrides.udeb:
 			                    into->overrides.deb,
 	                        file->name);
@@ -1280,6 +1284,16 @@ static retvalue prepare_deb(const struct incoming *i, const struct candidate *c,
 	if (RET_WAS_ERROR(r))
 		return r;
 
+	if (file->type == fe_DDEB &&
+	    !atomlist_in(&into->ddebcomponents, package->component)) {
+		fprintf(stderr,
+"Cannot put file '%s' of '%s' into component '%s',\n"
+"as it is not listed in DDebComponents of '%s'!\n",
+			BASENAME(i, file->ofs), BASENAME(i, c->ofs),
+			atoms_components[package->component],
+			into->codename);
+		return RET_ERROR;
+	}
 	if (file->type == fe_UDEB &&
 	    !atomlist_in(&into->udebcomponents, package->component)) {
 		fprintf(stderr,
@@ -1702,6 +1716,7 @@ static retvalue prepare_for_distribution(const struct incoming *i, const struct
 		switch (file->type) {
 			case fe_UDEB:
 			case fe_DEB:
+			case fe_DDEB:
 				r = prepare_deb(i, c, d, file);
 				break;
 			case fe_DSC:
diff --git a/main.c b/main.c
index f0a28163..0de6ae48 100644
--- a/main.c
+++ b/main.c
@@ -2615,7 +2615,9 @@ static retvalue repair_descriptions(struct target *target) {
 	retvalue result, r;
 
 	assert(target->packages == NULL);
-	assert(target->packagetype == pt_deb || target->packagetype == pt_udeb);
+	assert(target->packagetype == pt_deb ||
+			target->packagetype == pt_udeb ||
+			target->packagetype == pt_ddeb);
 
 	if (verbose > 2) {
 		printf(
@@ -2790,6 +2792,13 @@ ACTION_D(y, y, y, includedeb) {
 "Calling includeudeb with a -T not containing udeb makes no sense!\n");
 			return RET_ERROR;
 		}
+	} else if (strcmp(argv[0], "includeddeb") == 0) {
+		packagetype = pt_ddeb;
+		if (limitations_missed(packagetypes, pt_ddeb)) {
+			fprintf(stderr,
+"Calling includeddeb with a -T not containing ddeb makes no sense!\n");
+			return RET_ERROR;
+		}
 	} else if (strcmp(argv[0], "includedeb") == 0) {
 		packagetype = pt_deb;
 		if (limitations_missed(packagetypes, pt_deb)) {
@@ -2810,6 +2819,10 @@ ACTION_D(y, y, y, includedeb) {
 			if (!endswith(filename, ".udeb") && !IGNORING(extension,
 "includeudeb called with file '%s' not ending with '.udeb'\n", filename))
 				return RET_ERROR;
+		} else if (packagetype == pt_ddeb) {
+			if (!endswith(filename, ".ddeb") && !IGNORING(extension,
+"includeddeb called with file '%s' not ending with '.ddeb'\n", filename))
+				return RET_ERROR;
 		} else {
 			if (!endswith(filename, ".deb") && !IGNORING(extension,
 "includedeb called with file '%s' not ending with '.deb'\n", filename))
@@ -2832,6 +2845,8 @@ ACTION_D(y, y, y, includedeb) {
 		result = override_read(distribution->udeb_override,
 				&distribution->overrides.udeb, false);
 	else
+		/* we use the normal deb overrides for ddebs too -
+		 * they're not meant to have overrides anyway */
 		result = override_read(distribution->deb_override,
 				&distribution->overrides.deb, false);
 	if (RET_WAS_ERROR(result)) {
@@ -4103,6 +4118,8 @@ static const struct action {
 		2, -1, "[--delete] includedeb <distribution> <.deb-file>"},
 	{"includeudeb",		A_Dactsp(includedeb)|NEED_DELNEW,
 		2, -1, "[--delete] includeudeb <distribution> <.udeb-file>"},
+	{"includeddeb",		A_Dactsp(includedeb)|NEED_DELNEW,
+		2, -1, "[--delete] includeddeb <distribution> <.ddeb-file>"},
 	{"includedsc",		A_Dactsp(includedsc)|NEED_DELNEW,
 		2, 2, "[--delete] includedsc <distribution> <package>"},
 	{"include",		A_Dactsp(include)|NEED_DELNEW,
@@ -4295,7 +4312,7 @@ static retvalue callaction(command_t command, const struct action *action, int a
 			if (r == RET_NOTHING) {
 				fprintf(stderr,
 "Error: Packagetype '%s' as given to --packagetype is not know.\n"
-"(only dsc, deb, udeb and combinations of those are allowed)\n",
+"(only dsc, deb, udeb, ddeb and combinations of those are allowed)\n",
 					unknownitem);
 				r = RET_ERROR;
 			}
@@ -4562,7 +4579,7 @@ static void handle_option(int c, const char *argument) {
 " -P, --priority <priority>:         Force include* to set priority.\n"
 " -C, --component <component>: 	     Add,list or delete only in component.\n"
 " -A, --architecture <architecture>: Add,list or delete only to architecture.\n"
-" -T, --type <type>:                 Add,list or delete only type (dsc,deb,udeb).\n"
+" -T, --type <type>:                 Add,list or delete only type (dsc,deb,udeb,ddeb).\n"
 "\n"
 "actions (selection, for more see manpage):\n"
 " dumpreferences:    Print all saved references\n"
@@ -4581,6 +4598,8 @@ static void handle_option(int c, const char *argument) {
 "       Include the given upload.\n"
 " includedeb <distribution> <.deb-file>\n"
 "       Include the given binary package.\n"
+" includeddeb <distribution> <.ddeb-file>\n"
+"       Include the given debug binary package.\n"
 " includeudeb <distribution> <.udeb-file>\n"
 "       Include the given installer binary package.\n"
 " includedsc <distribution> <.dsc-file>\n"
diff --git a/pull.c b/pull.c
index 622477f5..3c5673df 100644
--- a/pull.c
+++ b/pull.c
@@ -60,6 +60,8 @@ struct pull_rule {
 	//e.g. "UDebComponents: main" // (not set means all)
 	struct atomlist udebcomponents;
 	bool udebcomponents_set;
+	// We don't have equivalents for ddebs yet since we don't know
+	// what the Debian archive layout is going to look like
 	// NULL means no condition
 	/*@null@*/term *includecondition;
 	struct filterlist filterlist;
diff --git a/target.c b/target.c
index c458ae79..2797b365 100644
--- a/target.c
+++ b/target.c
@@ -53,6 +53,10 @@ static char *calc_identifier(const char *codename, component_t component, archit
 		return mprintf("u|%s|%s|%s", codename,
 				atoms_components[component],
 				atoms_architectures[architecture]);
+	else if (packagetype == pt_ddeb)
+		return mprintf("d|%s|%s|%s", codename,
+				atoms_components[component],
+				atoms_architectures[architecture]);
 	else
 		return mprintf("%s|%s|%s", codename,
 				atoms_components[component],
@@ -131,6 +135,25 @@ retvalue target_initialize_ubinary(struct distribution *d, component_t component
 				atoms_architectures[architecture]),
 			exportmode, readonly, noexport, target);
 }
+retvalue target_initialize_dbinary(struct distribution *d, component_t component, architecture_t architecture, const struct exportmode *exportmode, bool readonly, bool noexport, const char *fakecomponentprefix, struct target **target) {
+	return target_initialize(d, component, architecture, pt_ddeb,
+			binaries_getversion,
+			binaries_getinstalldata,
+			binaries_getarchitecture,
+			binaries_getfilekeys, binaries_getchecksums,
+			binaries_getsourceandversion,
+			/* we use the main overrides */
+			binaries_doreoverride, binaries_retrack,
+			binaries_complete_checksums,
+			/* FIXME: we don't know what the Debian archive layout
+			 * is going to look like yet, so take a guess based
+			 * on udebs */
+			mprintf("%s/debug/binary-%s",
+				dist_component_name(component,
+					fakecomponentprefix),
+				atoms_architectures[architecture]),
+			exportmode, readonly, noexport, target);
+}
 retvalue target_initialize_binary(struct distribution *d, component_t component, architecture_t architecture, const struct exportmode *exportmode, bool readonly, bool noexport, const char *fakecomponentprefix, struct target **target) {
 	return target_initialize(d, component, architecture, pt_deb,
 			binaries_getversion,
diff --git a/target.h b/target.h
index 2664b614..32607ee6 100644
--- a/target.h
+++ b/target.h
@@ -77,6 +77,7 @@ struct target {
 };
 
 retvalue target_initialize_ubinary(/*@dependant@*/struct distribution *, component_t, architecture_t, /*@dependent@*/const struct exportmode *, bool /*readonly*/, bool /*noexport*/, /*@NULL@*/const char *fakecomponentprefix, /*@out@*/struct target **);
+retvalue target_initialize_dbinary(/*@dependant@*/struct distribution *, component_t, architecture_t, /*@dependent@*/const struct exportmode *, bool /*readonly*/, bool /*noexport*/, /*@NULL@*/const char *fakecomponentprefix, /*@out@*/struct target **);
 retvalue target_initialize_binary(/*@dependant@*/struct distribution *, component_t, architecture_t, /*@dependent@*/const struct exportmode *, bool /*readonly*/, bool /*noexport*/, /*@NULL@*/const char *fakecomponentprefix, /*@out@*/struct target **);
 retvalue target_initialize_source(/*@dependant@*/struct distribution *, component_t, /*@dependent@*/const struct exportmode *, bool /*readonly*/, bool /*noexport*/, /*@NULL@*/const char *fakecomponentprefix, /*@out@*/struct target **);
 retvalue target_free(struct target *);
diff --git a/tests/basic.sh b/tests/basic.sh
index f4a0f39c..867c489c 100755
--- a/tests/basic.sh
+++ b/tests/basic.sh
@@ -412,4 +412,14 @@ hello | 2.9-1 | buster-archive | $ARCH
 hello | 2.9-2 |         buster | $ARCH" "$($REPREPRO -b $REPO ls hello)"
 }
 
+test_ddeb() {
+	clear_distro
+	add_distro buster "DDebComponents: main non-free"
+	(cd $PKGS && PACKAGE=hello SECTION=main DISTRI=buster VERSION=2.9 REVISION=-1 DDEB=1 ../genpackage.sh)
+	#mv $PKGS/hello_2.9-1_${ARCH}.deb $PKGS/hello_2.9-1_${ARCH}.ddeb
+	#sed -i "s/hello_2.9-1_${ARCH}.deb/hello_2.9-1_${ARCH}.ddeb/g" $PKGS/hello_2.9-1_${ARCH}.changes
+	call $REPREPRO $VERBOSE_ARGS -b $REPO -C main include buster $PKGS/hello_2.9-1_${ARCH}.changes
+	assertEquals "hello | 2.9-1 | buster | $ARCH, source" "$($REPREPRO -b $REPO ls hello)"
+}
+
 . shunit2
diff --git a/tests/genpackage.sh b/tests/genpackage.sh
index d3516526..f228918b 100755
--- a/tests/genpackage.sh
+++ b/tests/genpackage.sh
@@ -9,6 +9,8 @@ if [ "x$OUTPUT" == "x" ] ; then
 fi
 
 DIR="$PACKAGE-$VERSION"
+ARCH="${ARCH:-$(dpkg-architecture -qDEB_HOST_ARCH)}"
+rm -rf "$DIR"
 mkdir "$DIR"
 mkdir "$DIR"/debian
 cat >"$DIR"/debian/control <<END
@@ -19,7 +21,7 @@ Maintainer: me <guess@who>
 Standards-Version: 0.0
 
 Package: $PACKAGE
-Architecture: ${ARCH:-$(dpkg-architecture -qDEB_HOST_ARCH)}
+Architecture: ${ARCH}
 Description: bla
  blub
 
@@ -28,6 +30,17 @@ Architecture: all
 Description: bla
  blub
 END
+
+if test -n "${DDEB-}" ; then
+cat >>"$DIR"/debian/control <<END
+
+Package: ${PACKAGE}-dbgsym
+Architecture: ${ARCH}
+Description: ${PACKAGE} debug symbols
+Package-Type: ddeb
+END
+fi
+
 if test -z "$DISTRI" ; then
 	DISTRI=test1
 fi
@@ -62,12 +75,26 @@ mkdir "$DIR"/debian/tmp/dir/subdir
 touch "$DIR"/debian/tmp/dir/subdir/file
 cd "$DIR"
 for pkg in `grep '^Package: ' debian/control | sed -e 's/^Package: //'` ; do
+	case "$pkg" in
+		(*-udeb)
+			deb="${pkg}_${VERSION}${REVISION}_${ARCH}.udeb"
+			;;
+		(*-dbgsym)
+			deb="${pkg}_${VERSION}${REVISION}_${ARCH}.ddeb"
+			;;
+		(*-addons)
+			deb="${pkg}_${FAKEVER:-${VERSION}${REVISION}}_all.deb"
+			;;
+		(*)
+			deb="${pkg}_${VERSION}${REVISION}_${ARCH}.deb"
+			;;
+	esac
 	if [ "x$pkg" != "x${pkg%-addons}" -a -n "$FAKEVER" ] ; then
 		dpkg-gencontrol -p$pkg -v"$FAKEVER"
 	else
 		dpkg-gencontrol -p$pkg
 	fi
-	dpkg --build debian/tmp .. > /dev/null
+	dpkg --build debian/tmp ../$deb > /dev/null
 done
 dpkg-genchanges -q "$@" > "$OUTPUT".pre
 # simulate dpkg-genchanges behaviour currently in sid so the testsuite runs for backports, too
diff --git a/tool.c b/tool.c
index 5716d44e..4c693822 100644
--- a/tool.c
+++ b/tool.c
@@ -101,7 +101,7 @@ static void binaryfile_free(struct binaryfile *p) {
 enum filetype { ft_UNKNOWN,
 			ft_TAR, ft_ORIG_TAR, ft_DIFF,
 #define ft_MaxInSource ft_DSC-1
-			ft_DSC, ft_DEB, ft_UDEB , ft_Count};
+			ft_DSC, ft_DEB, ft_UDEB, ft_DDEB, ft_Count};
 #define ft_Max ft_Count-1
 
 static const struct {
@@ -115,7 +115,8 @@ static const struct {
 	{ ".diff", 5, true},
 	{ ".dsc", 4, false},
 	{ ".deb", 4, false},
-	{ ".udeb", 5, false}
+	{ ".udeb", 5, false},
+	{ ".ddeb", 5, false}
 };
 
 struct dscfile {
@@ -199,7 +200,7 @@ static void fileentry_free(/*@only@*/struct fileentry *f) {
 	checksums_free(f->realchecksums);
 	free(f->section);
 	free(f->priority);
-	if (f->type == ft_DEB || f->type == ft_UDEB) {
+	if (f->type == ft_DEB || f->type == ft_DDEB || f->type == ft_UDEB) {
 		binaryfile_free(f->deb);
 	} else if (f->type == ft_DSC) {
 		dscfile_free(f->dsc);
@@ -947,7 +948,7 @@ static retvalue processfiles(const char *changesfilename, struct changes *change
 		if (RET_IS_OK(r)) {
 			if (file->type == ft_DSC)
 				r = parse_dsc(file, changes);
-			else if (file->type == ft_DEB || file->type == ft_UDEB)
+			else if (file->type == ft_DEB || file->type == ft_DDEB || file->type == ft_UDEB)
 				r = parse_deb(file, changes);
 			if (RET_WAS_ERROR(r)) {
 				free(dir);
@@ -958,7 +959,7 @@ static retvalue processfiles(const char *changesfilename, struct changes *change
 		if (r == RET_NOTHING) {
 			/* apply heuristics when not readable */
 			if (file->type == ft_DSC) {
-			} else if (file->type == ft_DEB || file->type == ft_UDEB) {
+			} else if (file->type == ft_DEB || file->type == ft_DDEB || file->type == ft_UDEB) {
 				struct binary *b; size_t len;
 
 				len = 0;
@@ -1819,7 +1820,7 @@ static retvalue verify(const char *changesfilename, struct changes *changes) {
 		const struct binary *b;
 		const struct binaryfile *deb;
 
-		if (file->type != ft_DEB && file->type != ft_UDEB)
+		if (file->type != ft_DEB && file->type != ft_DDEB && file->type != ft_UDEB)
 			continue;
 		if (file->fullfilename == NULL) {
 			fprintf(stderr,
@@ -2504,6 +2505,10 @@ static retvalue adddeb(struct changes *c, const char *debfilename, const struct
 			strcmp(fullfilename+strlen(fullfilename)-5, ".udeb") == 0) {
 		packagetype = "udeb";
 		type = ft_UDEB;
+	} else if (strlen(fullfilename) > 5 &&
+			strcmp(fullfilename+strlen(fullfilename)-5, ".ddeb") == 0) {
+		packagetype = "ddeb";
+		type = ft_DDEB;
 	} else {
 		packagetype = "deb";
 		type = ft_DEB;
@@ -2767,6 +2772,7 @@ static retvalue addfiles(const char *changesfilename, struct changes *c, int arg
 		size_t l = strlen(filename);
 
 		if ((l > 4 && strcmp(filename+l-4, ".deb") == 0) ||
+			(l > 5 && strcmp(filename+l-5, ".ddeb") == 0) ||
 		    (l > 5 && strcmp(filename+l-5, ".udeb") == 0))
 			r = adddeb(c, filename, searchpath);
 		else if ((l > 4 && strcmp(filename+l-4, ".dsc") == 0))
diff --git a/updates.c b/updates.c
index f6d7ada3..8d81ddfb 100644
--- a/updates.c
+++ b/updates.c
@@ -155,6 +155,9 @@ struct update_pattern {
 	// (empty means all)
 	struct strlist udebcomponents_from;
 	struct strlist udebcomponents_into;
+	// There's no ddeb support here yet, since we don't know what the
+	// Debian archive layout is going to look like.
+
 	// NULL means no condition
 	/*@null@*/term *includecondition;
 	struct filterlist filterlist;

From c2a497499da75b824a263baf22c7f907b5044f44 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 29 Mar 2017 02:10:05 +0200
Subject: [PATCH 66/70] Add trace debugging output

---
 checkindeb.c      |  8 ++++++++
 copypackages.c    |  8 ++++++++
 database.c        | 17 +++++++++++++++++
 distribution.c    | 12 ++++++++++++
 guesscomponent.c  |  7 +++++++
 printlistformat.c |  4 ++++
 reference.c       | 18 ++++++++++++++++++
 release.c         |  4 ++++
 target.c          | 28 ++++++++++++++++++++++++++++
 tracking.c        | 15 +++++++++++++++
 upgradelist.c     |  8 ++++++++
 11 files changed, 129 insertions(+)

diff --git a/checkindeb.c b/checkindeb.c
index 1b89cefd..00c0de47 100644
--- a/checkindeb.c
+++ b/checkindeb.c
@@ -105,6 +105,14 @@ static retvalue deb_preparelocation(struct debpackage *pkg, component_t forcecom
 	const struct overridedata *oinfo;
 	retvalue r;
 
+	if (verbose >= 15) {
+		fprintf(stderr, "trace: deb_preparelocation(pkg={deb={name=%s}}, forcecomponent=%s, forcearchitectures=[",
+		        pkg == NULL ? NULL : pkg->deb.name, atoms_components[forcecomponent]);
+		atomlist_fprint(stderr, at_architecture, forcearchitectures);
+		fprintf(stderr, "], forcesection=%s, packagetype=%s, distribution={codename=%s}, debfilename=%s) called.\n",
+		        forcesection, atoms_packagetypes[packagetype], distribution->codename, debfilename);
+	}
+
 	if (packagetype == pt_ddeb) {
 		/* ddebs don't have overrides */
 		forcesection = "debug";
diff --git a/copypackages.c b/copypackages.c
index 317627a2..428ce135 100644
--- a/copypackages.c
+++ b/copypackages.c
@@ -326,6 +326,10 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 	struct selectedpackage *package;
 	trackingdb tracks, fromtracks = NULL;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: packagelist_add(into.codename=%s, from.codename=%s) called.\n",
+		        into->codename, from != NULL ? from->codename : NULL);
+
 	r = distribution_prepareforwriting(into);
 	if (RET_WAS_ERROR(r))
 		return r;
@@ -354,6 +358,10 @@ static retvalue packagelist_add(struct distribution *into, const struct package_
 		struct target *target = tpl->target;
 		struct target *fromtarget = tpl->fromtarget;
 
+		if (verbose >= 15)
+			fprintf(stderr, "trace: Processing add/move from '%s' to '%s'...\n",
+			        fromtarget != NULL ? fromtarget->identifier : NULL, target->identifier);
+
 		r = target_initpackagesdb(target, READWRITE);
 		RET_ENDUPDATE(result, r);
 		if (RET_WAS_ERROR(r))
diff --git a/database.c b/database.c
index 49027434..7e8528c6 100644
--- a/database.c
+++ b/database.c
@@ -1064,6 +1064,9 @@ retvalue table_close(struct table *table) {
 	int dbret;
 	retvalue result = RET_OK;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: table_close(table.name=%s, table.subname=%s) called.\n",
+		        table == NULL ? NULL : table->name, table == NULL ? NULL : table->subname);
 	if (table == NULL)
 		return RET_NOTHING;
 	if (table->sec_berkeleydb != NULL) {
@@ -1374,6 +1377,9 @@ retvalue table_adduniqsizedrecord(struct table *table, const char *key, const ch
 	return RET_OK;
 }
 retvalue table_adduniqrecord(struct table *table, const char *key, const char *data) {
+	if (verbose >= 15)
+		fprintf(stderr, "trace: table_adduniqrecord(table={name: %s, subname: %s}, key=%s) called.\n",
+		        table->name, table->subname, key);
 	return table_adduniqsizedrecord(table, key, data, strlen(data)+1,
 			false, false);
 }
@@ -1410,6 +1416,9 @@ retvalue table_deleterecord(struct table *table, const char *key, bool ignoremis
 retvalue table_replacerecord(struct table *table, const char *key, const char *data) {
 	retvalue r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: table_replacerecord(table={name: %s, subname: %s}, key=%s) called.\n",
+		        table->name, table->subname, key);
 	r = table_deleterecord(table, key, false);
 	if (r != RET_ERROR_MISSING && RET_WAS_ERROR(r))
 		return r;
@@ -1421,6 +1430,10 @@ static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **c
 	struct cursor *cursor;
 	int dbret;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: newcursor(table={name: %s, subname: %s}) called.\n",
+		        table->name, table->subname);
+
 	if (table->sec_berkeleydb == NULL) {
 		berkeleydb = table->berkeleydb;
 	} else {
@@ -1840,6 +1853,10 @@ static retvalue database_table_secondary(const char *filename, const char *subta
 	struct opened_tables *opened_table;
 	retvalue r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: database_table_secondary(filename=%s, subtable=%s, type=%i, flags=%u, secondary_filename=%s, secondary_type=%i) called.\n",
+		        filename, subtable, type, flags, secondary_filename, secondary_type);
+
 	for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) {
 		if(strcmp2(iter->name, filename) == 0 && strcmp2(iter->subname, subtable) == 0) {
             fprintf(stderr,
diff --git a/distribution.c b/distribution.c
index 23c63b6c..11e2fe04 100644
--- a/distribution.c
+++ b/distribution.c
@@ -885,6 +885,9 @@ static retvalue export(struct distribution *distribution, bool onlyneeded) {
 	retvalue result, r;
 	struct release *release;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: export(distribution={codename: %s}, onlyneeded=%s)\n",
+		        distribution->codename, onlyneeded ? "true" : "false");
 	assert (distribution != NULL);
 
 	if (distribution->exportoptions[deo_noexport])
@@ -982,6 +985,8 @@ retvalue distribution_exportlist(enum exportwhen when, struct distribution *dist
 	bool todo = false;
 	struct distribution *d;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: distribution_exportlist() called.\n");
 	if (when == EXPORT_SILENT_NEVER) {
 		for (d = distributions ; d != NULL ; d = d->next) {
 			struct target *t;
@@ -1013,6 +1018,13 @@ retvalue distribution_exportlist(enum exportwhen when, struct distribution *dist
 
 	result = RET_NOTHING;
 	for (d=distributions; d != NULL; d = d->next) {
+		if (verbose >= 20)
+			fprintf(stderr, " looking at distribution {codename: %s, exportoptions[deo_noexport]: %s, omitted: %s, selected: %s, status: %d}.\n",
+			        d->codename,
+			        d->exportoptions[deo_noexport] ? "true" : "false",
+			        d->omitted ? "true" : "false",
+			        d->selected ? "true" : "false",
+			        d->status);
 		if (d->exportoptions[deo_noexport])
 			continue;
 		if (d->omitted || !d->selected)
diff --git a/guesscomponent.c b/guesscomponent.c
index adb3038a..8a13fc7e 100644
--- a/guesscomponent.c
+++ b/guesscomponent.c
@@ -36,6 +36,13 @@ retvalue guess_component(const char *codename, const struct atomlist *components
 	int i;
 	size_t section_len;
 
+	if (verbose >= 15) {
+		fprintf(stderr, "trace: guess_component(codename=%s, components=[", codename);
+		(void)atomlist_fprint(stderr, at_component, components);
+		fprintf(stderr, "], package=%s, section=%s, givencomponent=%s) called.\n",
+		        package, section, atoms_components[givencomponent]);
+	}
+
 	if (atom_defined(givencomponent)) {
 		if (!atomlist_in(components, givencomponent)) {
 			(void)fprintf(stderr,
diff --git a/printlistformat.c b/printlistformat.c
index 3b2d2e73..b1442bb1 100644
--- a/printlistformat.c
+++ b/printlistformat.c
@@ -38,6 +38,10 @@ retvalue listformat_print(const char *listformat, struct package *package) {
 	retvalue r;
 	const char *p, *q;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: listformat_print(package={name: %s, version: %s, pkgname: %s}) called.\n",
+		        package->name, package->version, package->pkgname);
+
 	if (listformat == NULL) {
 
 		r = package_getversion(package);
diff --git a/reference.c b/reference.c
index 5f59e9f4..ff1f0e7c 100644
--- a/reference.c
+++ b/reference.c
@@ -57,6 +57,10 @@ retvalue references_check(const char *referee, const struct strlist *filekeys) {
 retvalue references_increment(const char *needed, const char *neededby) {
 	retvalue r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: references_insert(needed=%s, neededby=%s) called.\n",
+		        needed, neededby);
+
 	r = table_addrecord(rdb_references, needed,
 			neededby, strlen(neededby), false);
 	if (RET_IS_OK(r) && verbose > 8)
@@ -95,6 +99,20 @@ retvalue references_insert(const char *identifier,
 	retvalue result, r;
 	int i;
 
+	if (verbose >= 15) {
+		fprintf(stderr, "trace: references_insert(identifier=%s, files=[", identifier);
+		for (i = 0 ; i < files->count ; i++) {
+			fprintf(stderr, "%s%s", i == 0 ? "" : ", ", files->values[i]);
+		}
+		fprintf(stderr, "], exclude=%s", exclude == NULL ? NULL : "[");
+		if (exclude != NULL) {
+			for (i = 0 ; i < exclude->count ; i++) {
+				fprintf(stderr, "%s%s", i == 0 ? "" : ", ", exclude->values[i]);
+			}
+		}
+		fprintf(stderr, "%s) called.\n", exclude == NULL ? "" : "]");
+	}
+
 	result = RET_NOTHING;
 
 	for (i = 0 ; i < files->count ; i++) {
diff --git a/release.c b/release.c
index 8dd953c4..d44872e2 100644
--- a/release.c
+++ b/release.c
@@ -175,6 +175,10 @@ retvalue release_init(struct release **release, const char *codename, const char
 	size_t len, suitelen, codenamelen;
 	retvalue r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: release_init(codename=%s, suite=%s, fakecomponentprefix=%s) called.\n",
+		        codename, suite, fakecomponentprefix);
+
 	n = zNEW(struct release);
 	if (FAILEDTOALLOC(n))
 		return RET_ERROR_OOM;
diff --git a/target.c b/target.c
index 2797b365..0984fc46 100644
--- a/target.c
+++ b/target.c
@@ -255,6 +255,9 @@ retvalue package_remove(struct package *old, struct logger *logger, struct track
 	assert (old->target != NULL && old->target->packages != NULL);
 
 	(void)package_getversion(old);
+	if (verbose >= 15)
+		fprintf(stderr, "trace: package_remove(old.name=%s, old.version=%s, old.target.identifier=%s) called.\n",
+		        old->name, old->version, old->target->identifier);
 	r = old->target->getfilekeys(old->control, &files);
 	if (RET_WAS_ERROR(r)) {
 		return r;
@@ -299,6 +302,9 @@ retvalue target_removepackage(struct target *target, struct logger *logger, cons
 	retvalue r;
 
 	assert(target != NULL && target->packages != NULL && name != NULL);
+	if (verbose >= 15)
+		fprintf(stderr, "trace: target_removepackage(target.identifier=%s, name=%s, version=%s) called.\n",
+		        target->identifier, name, version);
 
 	r = package_get(target, name, version, &old);
 	if (RET_WAS_ERROR(r)) {
@@ -377,6 +383,10 @@ static retvalue archive_package(struct target *target, const struct package *pac
 	bool close_database, close_trackingdb = false;
 	retvalue result, r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: archive_package(target.identifier=%s, package->name=%s, package->version=%s) called.\n",
+		        target->identifier, package->name, package->version);
+
 	if (target->distribution->archive != NULL) {
 		archive_target = distribution_gettarget(target->distribution->archive, target->component,
 		                                        target->architecture, target->packagetype);
@@ -450,6 +460,9 @@ static retvalue addpackages(struct target *target, const char *packagename, cons
 	struct table *table = target->packages;
 	enum filetype filetype;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: addpackages(target.identifier=%s, packagename=%s, version=%s, old->version=%s) called.\n",
+		        target->identifier, packagename, version, old != NULL ? old->version : NULL);
 	assert (atom_defined(architecture));
 
 	if (architecture == architecture_source)
@@ -524,6 +537,9 @@ retvalue target_addpackage(struct target *target, struct logger *logger, const c
 	struct package old;
 	retvalue r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: target_addpackage(target.identifier=%s, name=%s, version=%s) called.\n",
+		        target->identifier, name, version);
 	assert(target->packages!=NULL);
 
 	r = package_get(target, name, version, &old);
@@ -1067,6 +1083,10 @@ retvalue package_get(struct target *target, const char *name, const char *versio
 	retvalue result, r;
 	bool database_closed;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: package_get(target.identifier=%s, packagename=%s, version=%s) called.\n",
+		        target->identifier, name, version);
+
 	memset(pkg, 0, sizeof(*pkg));
 
 	database_closed = target->packages == NULL;
@@ -1105,6 +1125,10 @@ retvalue package_openiterator(struct target *t, bool readonly, bool duplicate, /
 	retvalue r, r2;
 	struct cursor *c;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: package_openiterator(target={identifier: %s}, readonly=%s, duplicate=%s) called.\n",
+		        t->identifier, readonly ? "true" : "false", duplicate ? "true" : "false");
+
 	tc->close_database = t->packages == NULL;
 	r = target_initpackagesdb(t, readonly);
 	assert (r != RET_NOTHING);
@@ -1153,6 +1177,10 @@ retvalue package_openduplicateiterator(struct target *t, const char *name, long
 
 bool package_next(struct package_cursor *tc) {
 	bool success;
+
+	if (verbose >= 15)
+		fprintf(stderr, "trace: package_next(tc={current: {name: %s, version: %s}}) called.\n", tc->current.name, tc->current.version);
+
 	package_done(&tc->current);
 	success = cursor_nexttempdata(tc->target->packages, tc->cursor,
 			&tc->current.name, &tc->current.control,
diff --git a/tracking.c b/tracking.c
index 473cc815..6fe55548 100644
--- a/tracking.c
+++ b/tracking.c
@@ -897,6 +897,9 @@ retvalue trackingdata_remove(struct trackingdata *data, const char* oldsource, c
 	retvalue result, r;
 	struct trackedpackage *pkg;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: trackingdata_remove(oldsource=%s, oldversion=%s) called.\n",
+		        oldsource, oldversion);
 	assert(oldsource != NULL && oldversion != NULL && oldfilekeys != NULL);
 	if (data->pkg != NULL &&
 			strcmp(oldversion, data->pkg->sourceversion) == 0 &&
@@ -1191,6 +1194,10 @@ static retvalue targetremovesourcepackage(trackingdb t, struct trackedpackage *p
 	const char *architecture = atoms_architectures[target->architecture];
 	const char *component = atoms_components[target->component];
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: targetremovesourcepackage(pkg={sourcename: %s, sourceversion: %s}, distribution.codename=%s, target.identifier=%s) called.\n",
+		        pkg->sourcename, pkg->sourceversion, distribution->codename, target->identifier);
+
 	result = RET_NOTHING;
 
 	component_len = strlen(component);
@@ -1345,6 +1352,10 @@ static retvalue removesourcepackage(trackingdb t, struct trackedpackage *pkg, st
 	retvalue result, r;
 	int i;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: removesourcepackage(pkg={sourcename: %s, sourceversion: %s}, distribution={codename: %s}) called.\n",
+		        pkg->sourcename, pkg->sourceversion, distribution->codename);
+
 	result = RET_NOTHING;
 	for (target = distribution->targets ; target != NULL ;
 	                                      target = target->next) {
@@ -1385,6 +1396,10 @@ retvalue tracking_removepackages(trackingdb t, struct distribution *distribution
 	struct trackedpackage *pkg;
 	retvalue result, r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: tracking_removepackages(distribution={codename: %s}, sourcename=%s, version=%s) called.\n",
+		        distribution->codename, sourcename, version);
+
 	if (version == NULL)
 		return tracking_foreachversion(t, distribution,
 				sourcename, removesourcepackage);
diff --git a/upgradelist.c b/upgradelist.c
index a02c44ff..96344235 100644
--- a/upgradelist.c
+++ b/upgradelist.c
@@ -99,6 +99,10 @@ static retvalue save_package_version(struct upgradelist *upgrade, struct package
 	if (RET_WAS_ERROR(r))
 		return r;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: save_package_version(upgrade.target={identifier: %s}, pkg={name: %s, version: %s, pkgname: %s}) called.\n",
+		        upgrade->target == NULL ? NULL : upgrade->target->identifier, pkg->name, pkg->version, pkg->pkgname);
+
 	package = zNEW(struct package_data);
 	if (FAILEDTOALLOC(package))
 		return RET_ERROR_OOM;
@@ -143,6 +147,10 @@ retvalue upgradelist_initialize(struct upgradelist **ul, struct target *t) {
 	retvalue r, r2;
 	struct package_cursor iterator;
 
+	if (verbose >= 15)
+		fprintf(stderr, "trace: upgradelist_initialize(target={identifier: %s}) called.\n",
+		        t == NULL ? NULL : t->identifier);
+
 	upgrade = zNEW(struct upgradelist);
 	if (FAILEDTOALLOC(upgrade))
 		return RET_ERROR_OOM;

From d7f9b260d65727ea610f224a57605367237974cc Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Tue, 2 Apr 2019 16:59:41 +0200
Subject: [PATCH 67/70] debian: Update changelog

Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 debian/changelog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 8e16eec6..e248235a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+reprepro (5.3.0-1.1) UNRELEASED; urgency=medium
+
+  * Non-maintainer upload.
+  * Switch to dh 9
+  * Support multiple versions. (Closes: #570623)
+
+ -- Benjamin Drung <benjamin.drung@cloud.ionos.com>  Tue, 02 Apr 2019 16:59:11 +0200
+
 reprepro (5.3.0-1) unstable; urgency=medium
 
   * new release

From d18e2ed95520e34aca4f7f84c1364c2cac4da51e Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Mon, 27 Aug 2018 15:56:35 +0200
Subject: [PATCH 68/70] Add README.md describing this git branch

---
 README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 00000000..8c125633
--- /dev/null
+++ b/README.md
@@ -0,0 +1,54 @@
+reprepro with multiple versions support
+=======================================
+
+This git repository hosts a branch for reprepro which adds multiple versions
+support to it. See [Debian bug #570623](https://bugs.debian.org/570623) for
+details and updates.
+
+The upstream repository can be found on https://salsa.debian.org/brlink/reprepro
+
+Release Notes
+=============
+
+The multiple-versions patch set adds following features:
+
+* Add shunit2 based tests (Closes: [#857302](https://bugs.debian.org/857302))
+* Support multiple versions. (Closes: [#570623](https://bugs.debian.org/570623))
+* Add the commands move, movesrc, movematched, movefilter
+* Add Limit and Archive option
+
+Database layout changes
+-----------------------
+
+The database layout changes from the upstream release to the
+multiple versions patch set. The difference is as following:
+
+### upstream
+
+* packages.db maps "package name" to "control file" without duplicates
+* no packagenames.db
+
+### multiple versions
+
+* packages.db maps "package name|version" to "control file" without
+duplicates
+* packagenames.db maps "package name" to "package name|version"
+allowing duplicates and duplicates sorted by dpkg --compare-versions
+descending
+
+The first time the database is opened by reprepro with multiple versions
+support, the database will be upgraded from the upstream layout to the multiple
+versions layout. *Warning*: There is no way back (but could be done with a
+simple Python script)!
+
+Howto rebase
+============
+
+1. Rebase the `multiple-versions` branch on top of the updated upstream
+`master` branch and push it to https://salsa.debian.org/bdrung/reprepro/
+
+2. Refresh the `multiple-versions-debian` branch by taking the upstream
+`debian` branch. Apply patch `debian: Switch to dh` and
+`Run shunit2 tests on build time`. Cherry-pick all commits from
+`multiple-versions`. Then apply patch `Add trace debugging output`,
+`debian: Update changelog` and `Add README.md describing this git branch`.

From 630bb1eb472bc4970c1133955994a6a90e9ab2ef Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 19 Jul 2019 12:28:41 +0200
Subject: [PATCH 69/70] README.md: Add "How to keep multiple versions"

Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 8c125633..ac037856 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,14 @@ The multiple-versions patch set adds following features:
 * Add the commands move, movesrc, movematched, movefilter
 * Add Limit and Archive option
 
+How to keep multiple versions
+-----------------------------
+
+The default behavior of this reprepro is identical to upstream's version. To
+keep multiple versions of the same package in the archive, you have to set the
+`Limit` option to the desired maximum amount (or to 0 for unlimited). See the
+description in the man page for details.
+
 Database layout changes
 -----------------------
 

From f3baf942c3870b679f32608ca8a1347cd6954ddc Mon Sep 17 00:00:00 2001
From: Hugo Laloge <hugo.laloge@gmail.com>
Date: Mon, 24 May 2021 13:53:56 +0200
Subject: [PATCH 70/70] Accept ddeb from remote repositories when using
 "update"

---
 remoterepository.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/remoterepository.c b/remoterepository.c
index 3134ca4a..05ddd4d0 100644
--- a/remoterepository.c
+++ b/remoterepository.c
@@ -1542,6 +1542,13 @@ struct remote_index *remote_index(struct remote_distribution *rd, const char *ar
 		cachefilename = genlistsfilename("uPackages", 4,
 				rd->repository->name, rd->suite,
 				component, architecture, ENDOFARGUMENTS);
+        } else if (packagetype == pt_ddeb) {
+		filename_in_release = mprintf(
+"%s/debug/binary-%s/Packages",
+				component, architecture);
+		cachefilename = genlistsfilename("dPackages", 4,
+				rd->repository->name, rd->suite,
+				component, architecture, ENDOFARGUMENTS);
 	} else if (packagetype == pt_dsc) {
 		filename_in_release = mprintf(
 "%s/source/Sources",
@@ -1564,6 +1571,10 @@ void cachedlistfile_need_index(struct cachedlistfile *list, const char *reposito
 		cachedlistfile_need(list, "uPackages", 4,
 				repository, suite,
 				component, architecture, ENDOFARGUMENTS);
+        } else if (packagetype == pt_ddeb) {
+		cachedlistfile_need(list, "dPackages", 4,
+				repository, suite,
+				component, architecture, ENDOFARGUMENTS);
 	} else if (packagetype == pt_dsc) {
 		cachedlistfile_need(list, "Sources", 3,
 				repository, suite,
@@ -1580,6 +1591,11 @@ struct remote_index *remote_flat_index(struct remote_distribution *rd, packagety
 		cachefilename = genlistsfilename("Packages", 2,
 				rd->repository->name, rd->suite,
 				ENDOFARGUMENTS);
+        } else if (packagetype == pt_ddeb) {
+                filename_in_release = strdup("dPackages");
+                cachefilename = genlistsfilename("dPackages", 2,
+				rd->repository->name, rd->suite,
+				ENDOFARGUMENTS);
 	} else if (packagetype == pt_dsc) {
 		filename_in_release = strdup("Sources");
 		cachefilename = genlistsfilename("Sources", 2,
@@ -1595,6 +1611,9 @@ void cachedlistfile_need_flat_index(struct cachedlistfile *list, const char *rep
 	if (packagetype == pt_deb) {
 		cachedlistfile_need(list, "Packages", 2,
 				repository, suite, ENDOFARGUMENTS);
+        } else if (packagetype == pt_ddeb) {
+		cachedlistfile_need(list, "dPackages", 2,
+				repository, suite, ENDOFARGUMENTS);
 	} else if (packagetype == pt_dsc) {
 		cachedlistfile_need(list, "Sources", 1,
 				repository, suite, ENDOFARGUMENTS);