testing v0.4.7.1

.github/workflows/release.yml

@@ -68,22 +68,28 @@ jobs:
       - name: Build all binaries
         run: |
           VERSION=$(cat appVersion.txt)
-
+          #
+          # disabled binaries:
+          #"merge-active:merge-active"
+          #"merge-descriptions:merge-descriptions"
+          #"test-MsgIdItemCache:test-MsgIdItemCache"
+          #"history-rebuild:history-rebuild"
+          #"fix-references:fix-references"
+          #
+          #
           # List of all applications to build
           apps=(
             "web:webserver"
             "nntp-fetcher:pugleaf-fetcher"
             "nntp-server:pugleaf-nntp-server"
             "expire-news:expire-news"
-            "merge-active:merge-active"
-            "merge-descriptions:merge-descriptions"
-            "test-MsgIdItemCache:test-MsgIdItemCache"
-            "history-rebuild:history-rebuild"
-            "fix-references:fix-references"
             "fix-thread-activity:fix-thread-activity"
             "rslight-importer:rslight-importer"
             "nntp-analyze:nntp-analyze"
             "recover-db:recover-db"
+            "nntp-transfer:nntp-transfer"
+            "post-queue:post-queue"
+            "tcp2tor:tcp2tor"
           )
 
           for app in "${apps[@]}"; do
@@ -114,12 +120,12 @@ jobs:
           OS="${{ matrix.os }}"
           ARCH="${{ matrix.arch }}"
           CHECKSUMS_FILE="checksums-${OS}-${ARCH}.sha256"
-          
+
           echo "Generating SHA256 checksums for ${OS}-${ARCH} binaries..."
           cd build
           sha256sum * > "../${CHECKSUMS_FILE}"
           cd ..
-          
+
           echo "CHECKSUMS_FILE=${CHECKSUMS_FILE}" >> $GITHUB_ENV
           echo ""
           echo "Generated checksums for ${OS}-${ARCH}:"
@@ -132,10 +138,10 @@ jobs:
           OS="${{ matrix.os }}"
           ARCH="${{ matrix.arch }}"
           ARCHIVE_NAME="go-pugleaf-v${VERSION}-${OS}-${ARCH}"
-          
+
           # Include checksums in the archive
           cp "${{ env.CHECKSUMS_FILE }}" build/checksums.sha256
-          
+
           if [ "${{ matrix.os }}" = "windows" ]; then
             zip -r "${ARCHIVE_NAME}.zip" build/ README.md LICENSE
             echo "ARCHIVE_FILE=${ARCHIVE_NAME}.zip" >> $GITHUB_ENV
@@ -184,13 +190,13 @@ jobs:
           echo "# Go-Pugleaf v${{ steps.version.outputs.version }} - SHA256 Checksums" > SHA256SUMS.txt
           echo "# Generated on $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> SHA256SUMS.txt
           echo "" >> SHA256SUMS.txt
-          
+
           # Process each platform's checksums
           for artifact_dir in artifacts/go-pugleaf-*; do
             if [ -d "$artifact_dir" ]; then
               platform=$(basename "$artifact_dir" | sed 's/go-pugleaf-//')
               echo "## Platform: $platform" >> SHA256SUMS.txt
-              
+
               # Find and process checksums file for this platform
               checksums_file=$(find "$artifact_dir" -name "checksums-*.sha256" | head -1)
               if [ -f "$checksums_file" ]; then
@@ -205,7 +211,7 @@ jobs:
               echo "" >> SHA256SUMS.txt
             fi
           done
-          
+
           echo "Comprehensive checksums file created:"
           echo "====================================="
           cat SHA256SUMS.txt
@@ -225,15 +231,20 @@ jobs:
             - `pugleaf-fetcher` - Article fetcher from NNTP providers
             - `pugleaf-nntp-server` - NNTP server implementation
             - `expire-news` - Article expiration tool
-            - `merge-active` - Active file merger
-            - `merge-descriptions` - Description file merger
-            - `test-MsgIdItemCache` - Cache testing tool
-            - `history-rebuild` - History rebuild utility
-            - `fix-references` - Reference fixing tool
             - `fix-thread-activity` - Thread activity fixer
             - `rslight-importer` - RSLight data importer
             - `nntp-analyze` - NNTP analysis tool
             - `recover-db` - Database recovery tool
+            - `nntp-transfer` - Tool to transfer newsgroups
+            - `post-queue` - Tool to send out queued posts
+            - `tcp2tor` - TCP to Tor proxy
+
+            ## NOT included:
+            - `fix-references` - Reference fixing tool
+            - `history-rebuild` - History rebuild utility
+            - `merge-active` - Active file merger
+            - `merge-descriptions` - Description file merger
+            - `test-MsgIdItemCache` - Cache testing tool
 
             ### Platform Support:
             - Linux (amd64, arm64)
@@ -250,7 +261,7 @@ jobs:
             ```bash
             # Verify individual platform binaries
             sha256sum -c checksums-linux-amd64.sha256
-            
+
             # Verify after archive extraction
             tar -xzf go-pugleaf-v${{ steps.version.outputs.version }}-linux-amd64.tar.gz
             cd build/

.gitignore

@@ -28,6 +28,7 @@ go.work
 /new*
 /demo_history
 internal/nntp/analysis_*
+getUpdate.sh
 update.tar.gz
 checksums.sha256
 checksums.sha256.archive
@@ -37,3 +38,5 @@ active_files/local-mode.active.new
 active_files/local-mode.active.new.sorted
 active_files/local-mode.active
 /old_overview
+cmd/tcp2tor/tcp2tor
+cmd/tcp2tor/tcp2tor.sha256

Build_DEV.sh

@@ -14,6 +14,7 @@ rm -v build/*
 #./build_analyze.sh
 ./build_fetcher.sh
 ./build_webserver.sh
+./build_nntp-transfer.sh
 #./build_recover-db.sh
 #./build_expire-news.sh
 

README.md

@@ -38,7 +38,11 @@ su pugleaf
 cd /home/pugleaf
 git clone https://github.com/go-while/go-pugleaf.git
 cd go-pugleaf
-git checkout testing-001
+
+# Warning: Nightly patches may be unstable or contain bugs
+# Only switch to the testing branch if you understand the risks or a dev told you to test a Patch!
+# git checkout testing-001
+ git checkout main
 
 # Build all binaries (outputs to ./build)
 ./build_ALL.sh && cp build/* .
@@ -65,7 +69,7 @@ mv build/usermgr .
   - Or bulk import newsgroups:
 ```bash
 ./webserver -import-active preload/active.txt -nntphostname your.domain.com
-./webserver -update-desc preload/newsgroups.descriptions -nntphostname your.domain.com
+./webserver -update-descr preload/newsgroups.descriptions -nntphostname your.domain.com
 ./rslight-importer -data data -etc etc -spool spool -nntphostname your.domain.com
 ```
   - rslight section import: see etc/menu.conf and creating sections aka folders in etc/ containing a groups.txt (e.g., etc/section/groups.txt)
@@ -243,11 +247,8 @@ go-pugleaf includes command-line applications for various newsgroup management t
 
 **Server Configuration:**
 - `-webport int` - Web server port (default: 11980 (no ssl) or 19443 (webssl))
-- `-withnntp` - Start NNTP server with default ports 1119/1563
 - `-nntptcpport int` - NNTP TCP port
 - `-nntptlsport int` - NNTP TLS port
-- `-withfetch` - Enable internal Cronjob to fetch new articles
-- `-isleep int` - Sleeps in fetch routines (default: 300 seconds = 5min)
 
 **Cache Configuration:**
 - `-maxsanartcache int` - Maximum number of cached sanitized articles (default: 10000)
@@ -262,10 +263,15 @@ go-pugleaf includes command-line applications for various newsgroup management t
 - `-import-desc string` - Import newsgroups from descriptions file
 - `-import-create` - Create missing newsgroups when importing
 - `-write-active-file string` - Write NNTP active file to specified path
+- `-write-active-only` - Use with -write-active-file (false writes only non active groups!) (default: true)
 - `-update-descr` - Update newsgroup descriptions from file
 - `-repair-watermarks` - Repair corrupted newsgroup watermarks
 - `-update-newsgroup-activity` - Update newsgroup activity timestamps
 - `-update-newsgroups-hide-futureposts` - Hide articles posted > 48h in future
+- `-compare-active string` - Compare active file with database and show missing groups (format: groupname highwater lowwater status)
+- `-compare-active-min-articles int64` - Use with -compare-active: only show groups with more than N articles (calculated as high-low)
+- `-rsync-inactive-groups string` - Path to new data dir, uses rsync to copy all inactive group databases to new data folder
+- `-rsync-remove-source` - Use with -rsync-inactive-groups. If set, removes source files after moving inactive groups (default: false)
 
 **Bridge Features: (NOT WORKING!)**
 - `-enable-fediverse` - Enable Fediverse bridge
@@ -277,8 +283,13 @@ go-pugleaf includes command-line applications for various newsgroup management t
 - `-matrix-userid string` - Matrix user ID
 
 **Advanced Options:**
-- `-useshorthashlen int` - Short hash length for history storage (2-7, default: 7)
-- `-ignore-initial-tiny-groups int` - Ignore tiny groups with fewer articles (default: 0)
+- `-useshorthashlen int` - Short hash length for history storage (2-7, default: 7) - NOTE: cannot be changed once set!
+
+**Disabled Options (commented out in code):**
+- `# -withnntp` - Start NNTP server with default ports 1119/1563
+- `# -withfetch` - Enable internal Cronjob to fetch new articles
+- `# -isleep int` - Sleeps in fetch routines (default: 300 seconds = 5min)
+- `# -ignore-initial-tiny-groups int` - Ignore tiny groups with fewer articles (default: 0)
 
 #### `pugleaf-fetcher` (cmd/nntp-fetcher)
 **Article fetcher from NNTP providers**
@@ -289,15 +300,6 @@ go-pugleaf includes command-line applications for various newsgroup management t
 **Required Flags:**
 - `-nntphostname string` - Your hostname must be set!
 
-**Connection Configuration:**
-- `-host string` - NNTP hostname (default: "81-171-22-215.pugleaf.net")
-- `-port int` - NNTP port (default: 563)
-- `-username string` - NNTP username (default: "read")
-- `-password string` - NNTP password (default: "only")
-- `-ssl` - Use SSL/TLS connection (default: true)
-- `-timeout int` - Connection timeout in seconds (default: 30)
-- `-message-id string` - Test specific message ID
-
 **Fetching Options:**
 - `-group string` - Newsgroup to fetch (empty = all groups or wildcard like rocksolid.*)
 - `-download-start-date string` - Start downloading from date (YYYY-MM-DD format)
@@ -307,10 +309,13 @@ go-pugleaf includes command-line applications for various newsgroup management t
 - `-max-batch int` - Maximum articles per batch (default: 128, recommended: 100)
 - `-max-batch-threads int` - Concurrent newsgroup batches (default: 16)
 - `-max-loops int` - Loop a group this many times (default: 1)
+- `-max-queue int` - Limit db_batch to have max N articles queued over all newsgroups (default: 16384)
 - `-download-max-par int` - Groups in parallel (default: 1, can consume memory!)
 - `-useshorthashlen int` - Short hash length for history (2-7, default: 7)
 
 **Advanced Options:**
+- `-ignore-initial-tiny-groups int64` - Ignore tiny groups with fewer articles during initial fetch (default: 0)
+- `-update-newsgroups-from-remote string` - Fetch remote newsgroup list and add new groups (empty = disabled, use "group.*" or "$all")
 - `-xover-copy` - Copy xover data from remote server (default: false)
 - `-test-conn` - Test connection and exit (default: false)
 - `-help` - Show usage examples and exit

THREAD_REBUILD_IMPLEMENTATION.md

@@ -0,0 +1,114 @@
+# Thread Rebuild Implementation
+## Usage Examples
+
+### 1. Rebuild threads for a specific newsgroup
+
+```bash
+# Rebuild threads for comp.lang.go
+./recover-db -db /path/to/data -group comp.lang.go -rebuild-threads -v
+
+# Rebuild threads for all newsgroups
+./recover-db -db /path/to/data -group '$all' -rebuild-threads -v
+```
+
+### 2. Repair database with automatic thread rebuilding
+
+```bash
+# Check and repair with automatic thread rebuilding when needed
+./recover-db -db /path/to/data -group comp.lang.go -repair -v
+```
+
+### 3. Check consistency first, then rebuild if needed
+
+```bash
+# First check what needs fixing
+./recover-db -db /path/to/data -group comp.lang.go -v
+
+# Then rebuild threads if orphaned threads are found
+./recover-db -db /path/to/data -group comp.lang.go -rebuild-threads -v
+```
+
+## Process Flow
+
+### Thread Rebuild Process:
+
+1. **Validation**: Check newsgroup exists and has articles
+2. **Cleanup**: Delete all existing thread-related data:
+   - `tree_stats` table
+   - `cached_trees` table
+   - `thread_cache` table
+   - `threads` table
+3. **Mapping**: Build message-ID to article-number lookup table
+4. **Threading**: Process articles in batches:
+   - Parse References header for each article
+   - Identify thread roots (no references)
+   - Find parent articles and calculate depth
+   - Insert thread relationships
+5. **Reporting**: Generate comprehensive rebuild report
+
+
+## Technical Details
+
+### Memory Management
+- Processes articles in configurable batches (default: 25,000)
+- Builds message-ID mapping incrementally to handle large newsgroups
+- Releases resources after each batch
+
+### Error Handling
+- Continues processing even if individual articles fail
+- Reports all errors in final report
+- Transactions ensure database consistency
+- Graceful handling of missing references
+
+### Performance Optimizations
+- Efficient SQL queries with proper indexing
+- Batch inserts for thread relationships
+- Minimal memory footprint during processing
+- Progress reporting for long operations
+
+## Files Modified
+
+1. **`internal/database/db_rescan.go`**:
+   - Added `RebuildThreadsFromScratch()` function
+   - Added `processThreadBatch()` helper function
+   - Added `parseReferences()` utility function
+   - Added `ThreadRebuildReport` struct and methods
+   - Added necessary imports (strings, time)
+
+2. **`cmd/recover-db/main.go`**:
+   - Added `--rebuild-threads` command line flag
+   - Added standalone thread rebuild mode
+   - Added automatic thread rebuilding after repairs
+   - Enhanced help text and output formatting
+
+## Future Enhancements
+
+1. **Performance improvements**:
+   - Parallel processing for very large newsgroups
+   - Configurable batch sizes
+   - Memory usage optimization
+
+2. **Additional features**:
+   - Incremental thread rebuilding (only rebuild changed threads)
+   - Thread validation without full rebuild
+   - Integration with web interface for manual triggering
+
+3. **Monitoring**:
+   - Thread rebuild scheduling
+   - Automatic detection of thread corruption
+   - Performance metrics and alerts
+
+## Testing
+
+Test the implementation with:
+
+```bash
+# Test with a small newsgroup first
+./recover-db -db ./data -group test.newsgroup -rebuild-threads -v
+
+# Test repair with thread rebuilding
+./recover-db -db ./data -group test.newsgroup -repair -v
+
+# Test with all newsgroups (use carefully!)
+./recover-db -db ./data -group '$all' -rebuild-threads -v
+```

appVersion.txt

@@ -1 +1 @@
-4.7.0
+4.7.1

build_ALL.sh

@@ -16,6 +16,9 @@ rm -v build/*
 ./build_webserver.sh
 ./build_recover-db.sh
 ./build_expire-news.sh
+./build_nntp-transfer.sh
+./build_post-queue.sh
+./build_tcp2tor.sh
 
 # Always generate checksums after building
 echo "Generating checksums for built executables..."

build_nntp-transfer.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "$0"
+GOEXPERIMENT=greenteagc go build -race -o build/nntp-transfer -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/nntp-transfer

build_post-queue.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "$0"
+GOEXPERIMENT=greenteagc go build -race -o build/post-queue -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/post-queue
+exit $?

build_recover-db.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 echo "$0"
-go build -o build/recover-db -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/recover-db
+GOEXPERIMENT=greenteagc go build -o build/recover-db -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/recover-db

build_tcp2tor.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "$0"
+GOEXPERIMENT=greenteagc go build -race -o build/tcp2tor -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/tcp2tor/
+exit $?