restucture cache; recursive pin; html-list for archive;

-cache structure has changed, normal pkgs are now in cache/ and additionally as copy in the individually repo folders
-root folders are now recursively pinned, this should allow for faster lookup, sinc the whole cluster hold all data (for directories)
-the archive is now not a copy, but only a html-list. This allows the importing server to properly garbage collect after 2 month of holding packages (they are no longer stored in the MFS) fixes #1

README.md

@@ -1,7 +1,4 @@
-# NOTICE: Temporary outage because of rollout of a new version
-
-
-# pacman.store 
+# pacman.store
 
 The domain [pkg.pacman.store](http://pkg.pacman.store) offers pkgs for Pacman you can mount to access it as Pacman cache through your local IPFS daemon. All files accessed this way will be downloaded to your IPFS cache and shared with the network.
 
@@ -42,9 +39,9 @@ ISO and bootstrap files are also stored on the cluster:
 
 *Before you use a pkg, make sure to read the WARNING.txt!*
 
-We hold old pkgs (disappeared from the mirrors) for another 2 month as snapshots in the cluster. Corresponding db snapshots can be found in the *db* subfolder. Older pkgs might not be accessible any longer. This depends on the garbage collections on the cluster-members and how long other clients in the network hold the files.
+We hold old pkgs (disappeared from the mirrors) for another 2 month as snapshots in the cluster. Corresponding db snapshots can be found in the *db* subfolder. Older pkgs or snapshots might not be accessible any longer. This depends on the garbage collections on the cluster-members and how long other clients in the network hold the files.
 
-If you want to keep a snapshot indefinitely, feel free to do so. Just pin the timestamp-subfolder in your client.
+If you want to keep a snapshot indefinitely, feel free to do so. Just pin the CID listed in the list-file in your client.
 
 
 
@@ -58,7 +55,7 @@ If you want to keep a snapshot indefinitely, feel free to do so. Just pin the ti
 | `/ipns/cluster.pacman.store` | cluster setup domain which holds the default.json and mdns_enabled.json |
 | `/ipns/pkg.pacman.store/arch/x86_64/default/` | current Pacman pkgs for ArchLinux (all standard repos + testing/staging) |
 | `/ipns/pkg.pacman.store/arch/x86_64/default/db/` | current Pacman databases for ArchLinux  (all standard repos + testing/staging) |
-| `/ipns/old.pkg.pacman.store/{ISO-8601-Timestamp}/` | archive of Pacman pkgs for ArchLinux  (all standard repos + testing/staging) |
+| `/ipns/old.pkg.pacman.store/arch/x86_64/default` | list-file of snapshots of the Pacman pkgs for ArchLinux with ISO-8601-Timestamp (all standard repos + testing/staging/unstable) |
 | `/ipns/iso.pacman.store/arch/x86_64/default/` | current ArchLinux ISOs+bootstrap images |
 
 

pacman wrapper/pacman_ipfs_sync

@@ -52,17 +52,29 @@ repo_id='default'
 
 # argument variables
 SKIP_DB_SYNC=0
+SKIP_CACHE_WIPE=0
 
 # simple one argument decoding
 
 if [ -n "$1" ]; then
 	if [ "$1" == '--skip-db-sync' ]; then
 		SKIP_DB_SYNC=1
+	elif [ "$1" == '--db-sync' ]; then
+		SKIP_DB_SYNC=0
 	else
 		fail "unexpected first argument" 100
 	fi
 fi
 
+if [ -n "$2" ]; then
+	if [ "$2" == '--skip-cache-wipe' ]; then
+		SKIP_CACHE_WIPE=1
+	elif [ "$2" == '--cache-wipe' ]; then
+		SKIP_CACHE_WIPE=0
+	else
+		fail "unexpected first argument" 100
+	fi
+fi
 
 # local functions
 
@@ -100,7 +112,7 @@ id -u "$ipfs_user" > /dev/null 2>&1 || fail "configured local user '$ipfs_user'
 
 su "$ipfs_user" --login -c "which 'ipfs' > /dev/null 2>&1" || fail "ipfs is not installed for user '$ipfs_user'" 150
 [ "$(ps aux | grep "^${ipfs_user}" | grep '/usr/bin/ipfs daemon' | wc -l)" -gt 0 ] || fail "ipfs is not running for user '$ipfs_user'" 151
-[ "$(mount -l | grep "/dev/fuse on "$ipns_mount" type fuse" | wc -l)" -eq 1 ] || fail "ipns is not mounted on configured location" 152
+[ $(mount -l | grep "/dev/fuse on "$ipns_mount" type fuse" | wc -l) -eq 1 ] || fail "ipns is not mounted on configured location" 152
 
 # end of config check
 
@@ -114,36 +126,39 @@ fi
 
 cd "$pacman_cache"
 
-find . -maxdepth 1 -type f  -print0 | while IFS= read -r -d $'\0' filename; do
+while IFS= read -r -d $'\0' filename; do
 	[ -z "$filename" ] && continue
 	[[ "$filename" == *.part ]] && continue #skip partial downloads
+	[[ "$filename" == *.sig ]] && continue #skip signature files
 	echo "adding pkg from local cache to ipfs: $filename"
-	su "$ipfs_user" --login -c "ipfs add --silent --raw-leaves --wrap-with-directory --pin=0 \"$pacman_cache/$filename\" > /dev/null 2>&1" 
-	rm "$filename"
-done
+	su "$ipfs_user" --login -c "ipfs add --silent --raw-leaves --pin=0 \"$pacman_cache/$filename\" > /dev/null 2>&1"
+	if [ $SKIP_CACHE_WIPE -eq 0 ]; then
+		rm "$filename"
+	fi
+done < <(find . -maxdepth 1 -type f  -print0)
 
 [ $SKIP_DB_SYNC -eq 1 ] && exit 0
 
-printf "accessing dbs on ipfs://pkg.pacman.store, this might take a while..." 
+printf "accessing dbs on ipfs://pkg.pacman.store, this might take a while..."
 	cd "$ipns_mount/pkg.pacman.store/$dist_id/$arch_id/$repo_id/db/" >/dev/null 2>&1 || fail "could not access the configured subdirectory of /ipns/pkg.pacman.store" 200 -2n
 echo " completed."
 
 grep '^\[' < "$pacman_conf" | grep -v '^\[options\]$' | sed 's/\[//g' | sed 's/\]//g' | while IFS= read -r active_repo; do
 	[[ ! " ${repos_to_sync[*]} " =~ $active_repo ]] && continue
-	
+
 	old_db_fullpath="${pacman_db}/sync/${active_repo}.db"
 	new_db_filename="${active_repo}.db"
-	
+
 	[ ! -f "$old_db_fullpath" ] && fail "could not locate '$new_db_filename' in pacman's db path"
-	
+
 	if [ ! -f "$new_db_filename" ]; then
 		echo "Warning: could not locate db file '$new_db_filename' on ipfs, SKIPPING"
 		continue
 	fi
-	
+
 	old_size="$(du --apparent-size --block-size=1 "$old_db_fullpath" | awk '{print $1}')"
 	new_size="$(du --apparent-size --block-size=1 "$new_db_filename" | awk '{print $1}')" || fail "error while fetching filesize for '$new_db_filename' from ipfs"
-	
+
 	[ "$old_size" -eq "$new_size" ] && continue
 	printf "receiving '%s'..." "$new_db_filename"
 	cp -ax "$new_db_filename" "${old_db_fullpath}.part" || fail "error while receiving db file for '$new_db_filename' from ipfs" -n

tmp/setup_client.txt

@@ -18,7 +18,7 @@ bindfs
 ipfs
 
 # edit /etc/fuse.conf
-uncomment "user_allow_other" 
+uncomment "user_allow_other"
 
 # you might want to setup the following with a custom user like 'ipfs'
 
@@ -38,13 +38,12 @@ After=network.target
 
 [Service]
 ExecStart=
-ExecStart=/usr/bin/ipfs daemon --mount --enable-pubsub-experiment --enable-namesys-pubsub --enable-mplex-experiment 
+ExecStart=/usr/bin/ipfs daemon --enable-pubsub-experiment --enable-namesys-pubsub --enable-mplex-experiment
 #
 
 # custom options in service file:
 # - pubsub-experiment       = a service to exchange messages inside ipfs, subscription based (similar to multicast)
 # - namesys-pubsub          = uses pubsub messages to speed-up name-lookups (something similar to dns (with dnssec) on the internet)
-# - mount                   = automatically mounts ipfs and ipfn to the filesystem via fuse
 # - enable-mplex-experiment = FIXME: some pixie magic
 
 
@@ -57,9 +56,6 @@ ipfs init --empty-repo
 
 # change the default config for best performance:
 
-# enable support for large directories
-ipfs config --json Experimental.ShardingEnabled true
-
 # faster name resolution
 ipfs config Pubsub.Router gossipsub
 
@@ -69,9 +65,6 @@ ipfs config --json Experimental.PreferTLS true
 # use QUIC/ip if possible instead of tcp - allows for multiplexing and lower overhead
 ipfs config --json Experimental.QUIC true
 
-#FIXME: create a description
-ipfs config --json Mounts.FuseAllowOther true #FIXME: show how to change this
-
 # by default IPFS will search for other computers in your network and connect to them
 # in rare cases you might want to deactivate this by running:
 ipfs config --json Discovery.MDNS.Enabled false
@@ -89,17 +82,10 @@ ipfs config --json Datastore.StorageMax '"100GB"'
 # increase the GracePeriod (minimum time before a new connection gets terminated)
 ipfs config --json Swarm.ConnMgr.GracePeriod '"3m"'
 
-# enable nat traversal and relay services if you use NAT
+# enable nat traversal and relay services if you use NAT (and don't have any IPv6 connectivity)
 ipfs config --json Swarm.EnableAutoNATService 'true'
 ipfs config --json Swarm.EnableAutoRelay 'true'
 
-# change standard path for ipfn-fileaccess
-ipfs config --json Mounts.IPFS '"/mnt/ipfs"'
-
-# change standard path for ipfn-fileaccess
-ipfs config --json Mounts.IPNS '"/mnt/ipns"'
-
-
 ### time to start and enable the service ###
 
 systemctl enable --now ipfs@<username>
@@ -110,7 +96,7 @@ systemctl enable --now ipfs@<username>
 
 # you might need to customize the program's settings for your installation
 
-run as root: ./pacman_ipfs_sync --skip-db-sync
+run as root: ./pacman_ipfs_sync --skip-db-sync --cache-wipe
 
 # create a folder where pacman should access the ipfs
 sudo mkdir /mnt/pacman_cache/