diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f631797b..2dccebcf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,22 +68,28 @@ jobs: - name: Build all binaries run: | VERSION=$(cat appVersion.txt) - + # + # disabled binaries: + #"merge-active:merge-active" + #"merge-descriptions:merge-descriptions" + #"test-MsgIdItemCache:test-MsgIdItemCache" + #"history-rebuild:history-rebuild" + #"fix-references:fix-references" + # + # # List of all applications to build apps=( "web:webserver" "nntp-fetcher:pugleaf-fetcher" "nntp-server:pugleaf-nntp-server" "expire-news:expire-news" - "merge-active:merge-active" - "merge-descriptions:merge-descriptions" - "test-MsgIdItemCache:test-MsgIdItemCache" - "history-rebuild:history-rebuild" - "fix-references:fix-references" "fix-thread-activity:fix-thread-activity" "rslight-importer:rslight-importer" "nntp-analyze:nntp-analyze" "recover-db:recover-db" + "nntp-transfer:nntp-transfer" + "post-queue:post-queue" + "tcp2tor:tcp2tor" ) for app in "${apps[@]}"; do @@ -114,12 +120,12 @@ jobs: OS="${{ matrix.os }}" ARCH="${{ matrix.arch }}" CHECKSUMS_FILE="checksums-${OS}-${ARCH}.sha256" - + echo "Generating SHA256 checksums for ${OS}-${ARCH} binaries..." cd build sha256sum * > "../${CHECKSUMS_FILE}" cd .. - + echo "CHECKSUMS_FILE=${CHECKSUMS_FILE}" >> $GITHUB_ENV echo "" echo "Generated checksums for ${OS}-${ARCH}:" @@ -132,10 +138,10 @@ jobs: OS="${{ matrix.os }}" ARCH="${{ matrix.arch }}" ARCHIVE_NAME="go-pugleaf-v${VERSION}-${OS}-${ARCH}" - + # Include checksums in the archive cp "${{ env.CHECKSUMS_FILE }}" build/checksums.sha256 - + if [ "${{ matrix.os }}" = "windows" ]; then zip -r "${ARCHIVE_NAME}.zip" build/ README.md LICENSE echo "ARCHIVE_FILE=${ARCHIVE_NAME}.zip" >> $GITHUB_ENV @@ -184,13 +190,13 @@ jobs: echo "# Go-Pugleaf v${{ steps.version.outputs.version }} - SHA256 Checksums" > SHA256SUMS.txt echo "# Generated on $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> SHA256SUMS.txt echo "" >> SHA256SUMS.txt - + # Process each platform's checksums for artifact_dir in artifacts/go-pugleaf-*; do if [ -d "$artifact_dir" ]; then platform=$(basename "$artifact_dir" | sed 's/go-pugleaf-//') echo "## Platform: $platform" >> SHA256SUMS.txt - + # Find and process checksums file for this platform checksums_file=$(find "$artifact_dir" -name "checksums-*.sha256" | head -1) if [ -f "$checksums_file" ]; then @@ -205,7 +211,7 @@ jobs: echo "" >> SHA256SUMS.txt fi done - + echo "Comprehensive checksums file created:" echo "=====================================" cat SHA256SUMS.txt @@ -225,15 +231,20 @@ jobs: - `pugleaf-fetcher` - Article fetcher from NNTP providers - `pugleaf-nntp-server` - NNTP server implementation - `expire-news` - Article expiration tool - - `merge-active` - Active file merger - - `merge-descriptions` - Description file merger - - `test-MsgIdItemCache` - Cache testing tool - - `history-rebuild` - History rebuild utility - - `fix-references` - Reference fixing tool - `fix-thread-activity` - Thread activity fixer - `rslight-importer` - RSLight data importer - `nntp-analyze` - NNTP analysis tool - `recover-db` - Database recovery tool + - `nntp-transfer` - Tool to transfer newsgroups + - `post-queue` - Tool to send out queued posts + - `tcp2tor` - TCP to Tor proxy + + ## NOT included: + - `fix-references` - Reference fixing tool + - `history-rebuild` - History rebuild utility + - `merge-active` - Active file merger + - `merge-descriptions` - Description file merger + - `test-MsgIdItemCache` - Cache testing tool ### Platform Support: - Linux (amd64, arm64) @@ -250,7 +261,7 @@ jobs: ```bash # Verify individual platform binaries sha256sum -c checksums-linux-amd64.sha256 - + # Verify after archive extraction tar -xzf go-pugleaf-v${{ steps.version.outputs.version }}-linux-amd64.tar.gz cd build/ diff --git a/.gitignore b/.gitignore index 9336fbf5..f40caef4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ go.work /new* /demo_history internal/nntp/analysis_* +getUpdate.sh update.tar.gz checksums.sha256 checksums.sha256.archive @@ -37,3 +38,5 @@ active_files/local-mode.active.new active_files/local-mode.active.new.sorted active_files/local-mode.active /old_overview +cmd/tcp2tor/tcp2tor +cmd/tcp2tor/tcp2tor.sha256 diff --git a/Build_DEV.sh b/Build_DEV.sh index 3fd99f3c..a6c7b0ff 100755 --- a/Build_DEV.sh +++ b/Build_DEV.sh @@ -14,6 +14,7 @@ rm -v build/* #./build_analyze.sh ./build_fetcher.sh ./build_webserver.sh +./build_nntp-transfer.sh #./build_recover-db.sh #./build_expire-news.sh diff --git a/README.md b/README.md index efcc7e46..53c84a6b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,11 @@ su pugleaf cd /home/pugleaf git clone https://github.com/go-while/go-pugleaf.git cd go-pugleaf -git checkout testing-001 + +# Warning: Nightly patches may be unstable or contain bugs +# Only switch to the testing branch if you understand the risks or a dev told you to test a Patch! +# git checkout testing-001 + git checkout main # Build all binaries (outputs to ./build) ./build_ALL.sh && cp build/* . @@ -65,7 +69,7 @@ mv build/usermgr . - Or bulk import newsgroups: ```bash ./webserver -import-active preload/active.txt -nntphostname your.domain.com -./webserver -update-desc preload/newsgroups.descriptions -nntphostname your.domain.com +./webserver -update-descr preload/newsgroups.descriptions -nntphostname your.domain.com ./rslight-importer -data data -etc etc -spool spool -nntphostname your.domain.com ``` - rslight section import: see etc/menu.conf and creating sections aka folders in etc/ containing a groups.txt (e.g., etc/section/groups.txt) @@ -243,11 +247,8 @@ go-pugleaf includes command-line applications for various newsgroup management t **Server Configuration:** - `-webport int` - Web server port (default: 11980 (no ssl) or 19443 (webssl)) -- `-withnntp` - Start NNTP server with default ports 1119/1563 - `-nntptcpport int` - NNTP TCP port - `-nntptlsport int` - NNTP TLS port -- `-withfetch` - Enable internal Cronjob to fetch new articles -- `-isleep int` - Sleeps in fetch routines (default: 300 seconds = 5min) **Cache Configuration:** - `-maxsanartcache int` - Maximum number of cached sanitized articles (default: 10000) @@ -262,10 +263,15 @@ go-pugleaf includes command-line applications for various newsgroup management t - `-import-desc string` - Import newsgroups from descriptions file - `-import-create` - Create missing newsgroups when importing - `-write-active-file string` - Write NNTP active file to specified path +- `-write-active-only` - Use with -write-active-file (false writes only non active groups!) (default: true) - `-update-descr` - Update newsgroup descriptions from file - `-repair-watermarks` - Repair corrupted newsgroup watermarks - `-update-newsgroup-activity` - Update newsgroup activity timestamps - `-update-newsgroups-hide-futureposts` - Hide articles posted > 48h in future +- `-compare-active string` - Compare active file with database and show missing groups (format: groupname highwater lowwater status) +- `-compare-active-min-articles int64` - Use with -compare-active: only show groups with more than N articles (calculated as high-low) +- `-rsync-inactive-groups string` - Path to new data dir, uses rsync to copy all inactive group databases to new data folder +- `-rsync-remove-source` - Use with -rsync-inactive-groups. If set, removes source files after moving inactive groups (default: false) **Bridge Features: (NOT WORKING!)** - `-enable-fediverse` - Enable Fediverse bridge @@ -277,8 +283,13 @@ go-pugleaf includes command-line applications for various newsgroup management t - `-matrix-userid string` - Matrix user ID **Advanced Options:** -- `-useshorthashlen int` - Short hash length for history storage (2-7, default: 7) -- `-ignore-initial-tiny-groups int` - Ignore tiny groups with fewer articles (default: 0) +- `-useshorthashlen int` - Short hash length for history storage (2-7, default: 7) - NOTE: cannot be changed once set! + +**Disabled Options (commented out in code):** +- `# -withnntp` - Start NNTP server with default ports 1119/1563 +- `# -withfetch` - Enable internal Cronjob to fetch new articles +- `# -isleep int` - Sleeps in fetch routines (default: 300 seconds = 5min) +- `# -ignore-initial-tiny-groups int` - Ignore tiny groups with fewer articles (default: 0) #### `pugleaf-fetcher` (cmd/nntp-fetcher) **Article fetcher from NNTP providers** @@ -289,15 +300,6 @@ go-pugleaf includes command-line applications for various newsgroup management t **Required Flags:** - `-nntphostname string` - Your hostname must be set! -**Connection Configuration:** -- `-host string` - NNTP hostname (default: "81-171-22-215.pugleaf.net") -- `-port int` - NNTP port (default: 563) -- `-username string` - NNTP username (default: "read") -- `-password string` - NNTP password (default: "only") -- `-ssl` - Use SSL/TLS connection (default: true) -- `-timeout int` - Connection timeout in seconds (default: 30) -- `-message-id string` - Test specific message ID - **Fetching Options:** - `-group string` - Newsgroup to fetch (empty = all groups or wildcard like rocksolid.*) - `-download-start-date string` - Start downloading from date (YYYY-MM-DD format) @@ -307,10 +309,13 @@ go-pugleaf includes command-line applications for various newsgroup management t - `-max-batch int` - Maximum articles per batch (default: 128, recommended: 100) - `-max-batch-threads int` - Concurrent newsgroup batches (default: 16) - `-max-loops int` - Loop a group this many times (default: 1) +- `-max-queue int` - Limit db_batch to have max N articles queued over all newsgroups (default: 16384) - `-download-max-par int` - Groups in parallel (default: 1, can consume memory!) - `-useshorthashlen int` - Short hash length for history (2-7, default: 7) **Advanced Options:** +- `-ignore-initial-tiny-groups int64` - Ignore tiny groups with fewer articles during initial fetch (default: 0) +- `-update-newsgroups-from-remote string` - Fetch remote newsgroup list and add new groups (empty = disabled, use "group.*" or "$all") - `-xover-copy` - Copy xover data from remote server (default: false) - `-test-conn` - Test connection and exit (default: false) - `-help` - Show usage examples and exit diff --git a/THREAD_REBUILD_IMPLEMENTATION.md b/THREAD_REBUILD_IMPLEMENTATION.md new file mode 100644 index 00000000..3965b2e2 --- /dev/null +++ b/THREAD_REBUILD_IMPLEMENTATION.md @@ -0,0 +1,114 @@ +# Thread Rebuild Implementation +## Usage Examples + +### 1. Rebuild threads for a specific newsgroup + +```bash +# Rebuild threads for comp.lang.go +./recover-db -db /path/to/data -group comp.lang.go -rebuild-threads -v + +# Rebuild threads for all newsgroups +./recover-db -db /path/to/data -group '$all' -rebuild-threads -v +``` + +### 2. Repair database with automatic thread rebuilding + +```bash +# Check and repair with automatic thread rebuilding when needed +./recover-db -db /path/to/data -group comp.lang.go -repair -v +``` + +### 3. Check consistency first, then rebuild if needed + +```bash +# First check what needs fixing +./recover-db -db /path/to/data -group comp.lang.go -v + +# Then rebuild threads if orphaned threads are found +./recover-db -db /path/to/data -group comp.lang.go -rebuild-threads -v +``` + +## Process Flow + +### Thread Rebuild Process: + +1. **Validation**: Check newsgroup exists and has articles +2. **Cleanup**: Delete all existing thread-related data: + - `tree_stats` table + - `cached_trees` table + - `thread_cache` table + - `threads` table +3. **Mapping**: Build message-ID to article-number lookup table +4. **Threading**: Process articles in batches: + - Parse References header for each article + - Identify thread roots (no references) + - Find parent articles and calculate depth + - Insert thread relationships +5. **Reporting**: Generate comprehensive rebuild report + + +## Technical Details + +### Memory Management +- Processes articles in configurable batches (default: 25,000) +- Builds message-ID mapping incrementally to handle large newsgroups +- Releases resources after each batch + +### Error Handling +- Continues processing even if individual articles fail +- Reports all errors in final report +- Transactions ensure database consistency +- Graceful handling of missing references + +### Performance Optimizations +- Efficient SQL queries with proper indexing +- Batch inserts for thread relationships +- Minimal memory footprint during processing +- Progress reporting for long operations + +## Files Modified + +1. **`internal/database/db_rescan.go`**: + - Added `RebuildThreadsFromScratch()` function + - Added `processThreadBatch()` helper function + - Added `parseReferences()` utility function + - Added `ThreadRebuildReport` struct and methods + - Added necessary imports (strings, time) + +2. **`cmd/recover-db/main.go`**: + - Added `--rebuild-threads` command line flag + - Added standalone thread rebuild mode + - Added automatic thread rebuilding after repairs + - Enhanced help text and output formatting + +## Future Enhancements + +1. **Performance improvements**: + - Parallel processing for very large newsgroups + - Configurable batch sizes + - Memory usage optimization + +2. **Additional features**: + - Incremental thread rebuilding (only rebuild changed threads) + - Thread validation without full rebuild + - Integration with web interface for manual triggering + +3. **Monitoring**: + - Thread rebuild scheduling + - Automatic detection of thread corruption + - Performance metrics and alerts + +## Testing + +Test the implementation with: + +```bash +# Test with a small newsgroup first +./recover-db -db ./data -group test.newsgroup -rebuild-threads -v + +# Test repair with thread rebuilding +./recover-db -db ./data -group test.newsgroup -repair -v + +# Test with all newsgroups (use carefully!) +./recover-db -db ./data -group '$all' -rebuild-threads -v +``` \ No newline at end of file diff --git a/appVersion.txt b/appVersion.txt index f6cdf409..7c66fca5 100644 --- a/appVersion.txt +++ b/appVersion.txt @@ -1 +1 @@ -4.7.0 +4.7.1 diff --git a/build_ALL.sh b/build_ALL.sh index c7a2e911..6784ab3b 100755 --- a/build_ALL.sh +++ b/build_ALL.sh @@ -16,6 +16,9 @@ rm -v build/* ./build_webserver.sh ./build_recover-db.sh ./build_expire-news.sh +./build_nntp-transfer.sh +./build_post-queue.sh +./build_tcp2tor.sh # Always generate checksums after building echo "Generating checksums for built executables..." diff --git a/build_nntp-transfer.sh b/build_nntp-transfer.sh new file mode 100755 index 00000000..b3ea4b78 --- /dev/null +++ b/build_nntp-transfer.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "$0" +GOEXPERIMENT=greenteagc go build -race -o build/nntp-transfer -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/nntp-transfer diff --git a/build_post-queue.sh b/build_post-queue.sh new file mode 100755 index 00000000..ff71eeac --- /dev/null +++ b/build_post-queue.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "$0" +GOEXPERIMENT=greenteagc go build -race -o build/post-queue -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/post-queue +exit $? diff --git a/build_recover-db.sh b/build_recover-db.sh index a0f54e3b..3dae9a54 100755 --- a/build_recover-db.sh +++ b/build_recover-db.sh @@ -1,3 +1,3 @@ #!/bin/bash echo "$0" -go build -o build/recover-db -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/recover-db +GOEXPERIMENT=greenteagc go build -o build/recover-db -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/recover-db diff --git a/build_tcp2tor.sh b/build_tcp2tor.sh new file mode 100755 index 00000000..92354f83 --- /dev/null +++ b/build_tcp2tor.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "$0" +GOEXPERIMENT=greenteagc go build -race -o build/tcp2tor -ldflags "-X main.appVersion=$(cat appVersion.txt)" ./cmd/tcp2tor/ +exit $? diff --git a/cmd/expire-news/main.go b/cmd/expire-news/main.go index a0ebf088..8ca7597c 100644 --- a/cmd/expire-news/main.go +++ b/cmd/expire-news/main.go @@ -63,8 +63,8 @@ func showUsageExamples() { func main() { config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("Starting go-pugleaf News Expiration Tool (version %s)", appVersion) - // Command line flags var ( targetGroup = flag.String("group", "", "Newsgroup to expire ('$all', specific group, or wildcard like news.*)") @@ -74,7 +74,6 @@ func main() { batchSize = flag.Int("batch-size", 1000, "Number of articles to process per batch") respectExpiry = flag.Bool("respect-expiry", false, "Use per-group expiry_days settings from database") prune = flag.Bool("prune", false, "Remove oldest articles to respect max_articles limit per group") - hostnamePath = flag.String("nntphostname", "", "Your hostname (required)") showHelp = flag.Bool("help", false, "Show usage examples and exit") ) flag.Parse() @@ -90,10 +89,6 @@ func main() { log.Fatal("Error: -group flag is required. Use -help for examples.") } - if *hostnamePath == "" { - log.Fatal("Error: -nntphostname flag is required") - } - if !*respectExpiry && *expireDays <= 0 && !*prune { log.Fatal("Error: -days must be > 0, or use -respect-expiry, or use -prune to use database settings") } @@ -107,10 +102,6 @@ func main() { log.Fatal("Error: Must specify -force to actually delete articles, or use -dry-run to preview") } - // Load config - mainConfig := config.NewDefaultConfig() - mainConfig.Server.Hostname = *hostnamePath - // Initialize database db, err := database.OpenDatabase(nil) if err != nil { diff --git a/cmd/history-rebuild/main.go b/cmd/history-rebuild/main.go index c624f63c..c722ba97 100644 --- a/cmd/history-rebuild/main.go +++ b/cmd/history-rebuild/main.go @@ -151,9 +151,13 @@ func main() { return } + // Set hostname in processor with database fallback support + if err := processor.SetHostname(*nntpHostname, db); err != nil { + log.Fatalf("Failed to set NNTP hostname: %v", err) + } + // Initialize processor with proper cache management fmt.Println("🔧 Initializing processor for cache management...") - processor.LocalHostnamePath = *nntpHostname proc := processor.NewProcessor(db, nil, lockedHashLen) // nil pool since we're not fetching // Set up the date parser adapter to use processor's ParseNNTPDate database.GlobalDateParser = processor.ParseNNTPDate diff --git a/cmd/nntp-analyze/main.go b/cmd/nntp-analyze/main.go index ddb4ed4c..d1a7981b 100644 --- a/cmd/nntp-analyze/main.go +++ b/cmd/nntp-analyze/main.go @@ -125,6 +125,7 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("Starting go-pugleaf NNTP Analyzer (version %s)", config.AppVersion) // Command line flags for NNTP server connection diff --git a/cmd/nntp-fetcher/main.go b/cmd/nntp-fetcher/main.go index 38594005..2907c758 100644 --- a/cmd/nntp-fetcher/main.go +++ b/cmd/nntp-fetcher/main.go @@ -51,30 +51,21 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion database.DBidleTimeOut = 15 * time.Second - database.FETCH_MODE = true // prevents booting caches + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("Starting go-pugleaf NNTP Fetcher (version %s)", config.AppVersion) // Command line flags for NNTP fetcher configuration var newsgroups []*models.Newsgroup var ( - host = flag.String("host", "81-171-22-215.pugleaf.net", "NNTP hostname") - port = flag.Int("port", 563, "NNTP port") - username = flag.String("username", "read", "NNTP username") - password = flag.String("password", "only", "NNTP password") - ssl = flag.Bool("ssl", true, "Use SSL/TLS connection") - timeout = flag.Int("timeout", 30, "Connection timeout in seconds") - testMsg = flag.String("message-id", "", "Test message ID to fetch (optional)") - maxBatchThreads = flag.Int("max-batch-threads", 16, "Limit how many newsgroup batches will be processed concurrently (default: 16)") - maxBatch = flag.Int("max-batch", 128, "Maximum number of articles to process in a batch (recommended: 100)") - maxLoops = flag.Int("max-loops", 1, "Loop a group this many times and fetch `-max-batch N` every loop") - ignoreInitialTinyGroups = flag.Int64("ignore-initial-tiny-groups", 0, "If > 0: initial fetch ignores tiny groups with fewer articles than this (default: 0)") - importOverview = flag.Bool("xover-copy", false, "Do not use xover-copy unless you want to Copy xover data from remote server and then articles. instead of normal 'xhdr message-id' --> articles (default: false)") - fetchNewsgroup = flag.String("group", "", "Newsgroup to fetch (default: empty = all groups once up to max-batch) or rocksolid.* with final wildcard to match prefix.*") - hostnamePath = flag.String("nntphostname", "", "Your hostname must be set!") - testConn = flag.Bool("test-conn", false, "Test direct connection to NNTP server and exit (default: false)") - useShortHashLenPtr = flag.Int("useshorthashlen", 7, "short hash length for history storage (2-7, default: 7) - NOTE: cannot be changed once set!") - fetchActiveOnly = flag.Bool("fetch-active-only", true, "Fetch only active newsgroups (default: true)") - downloadMaxPar = flag.Int("download-max-par", 1, "run this many groups in parallel, can eat your memory! (default: 1)") - updateList = flag.String("update-newsgroups-from-remote", "", "Fetch remote newsgroup list from first enabled provider and add new groups to database (default: empty, nothing. use \"group.*\" or \"\\$all\")") + maxBatchThreads = flag.Int("max-batch-threads", 4, "Limit how many newsgroup batches will be processed concurrently (default: 16) more can eat your memory and disk IO!") + maxBatch = flag.Int("max-batch", 1000, "Maximum number of articles to process in a batch (recommended: 100-10000)") + maxQueued = flag.Int("max-queue", 1280, "Limit db_batch to have max N articles queued over all newsgroups") + fetchNewsgroup = flag.String("group", "", "Newsgroup to fetch (default: empty = all groups once up to max-batch) or rocksolid.* with final wildcard to match prefix.*") + nntphostname = flag.String("nntphostname", "", "Your hostname must be set!") + useShortHashLenPtr = flag.Int("useshorthashlen", 7, "short hash length for history storage (2-7, default: 7) - NOTE: cannot be changed once set!") + fetchActiveOnly = flag.Bool("fetch-active-only", true, "Fetch only active newsgroups (default: true)") + downloadMaxPar = flag.Int("download-max-par", 1, "run this many groups in parallel, can eat your memory! (default: 1)") + updateList = flag.String("fetch-newsgroups-from-remote", "", "Fetch remote newsgroup list from first enabled provider (default: empty, nothing. use \"group.*\" or \"\\$all\")") + updateListForce = flag.Bool("fetch-newsgroups-force", false, "use with -fetch-newsgroups-from-remote .. to really add them to database") // Download options with date filtering downloadStartDate = flag.String("download-start-date", "", "Start downloading articles from this date (YYYY-MM-DD format)") showHelp = flag.Bool("help", false, "Show usage examples and exit") @@ -85,14 +76,8 @@ func main() { showUsageExamples() os.Exit(0) } - if *testConn { - if err := ConnectionTest(host, port, username, password, ssl, timeout, *fetchNewsgroup, testMsg); err != nil { - log.Fatalf("Connection test failed: %v", err) - } - os.Exit(0) - } if *updateList != "" { - if err := UpdateNewsgroupList(host, port, username, password, ssl, timeout, updateList); err != nil { + if err := UpdateNewsgroupList(updateList, *updateListForce); err != nil { log.Fatalf("Newsgroup list update failed: %v", err) } os.Exit(0) @@ -101,40 +86,33 @@ func main() { if *downloadMaxPar < 1 { *downloadMaxPar = 1 } - if *maxBatch < 1 { - *maxBatch = 1 + if *maxBatch < 10 { + *maxBatch = 10 } - if *maxLoops != 1 { - *maxLoops = 1 // hardcoded to 1 TODO find fixme + if *maxBatchThreads < 1 { + *maxBatchThreads = 1 } - if *maxBatch > 1000 { - log.Printf("[WARN] max batch: %d (should be between 100 and 1000)", *maxBatch) + if *maxQueued < 1 { + *maxQueued = 1 } - if *maxBatch > 100000 { - log.Printf("[WARN] max batch can not be higher than 100000") - *maxBatch = 100000 + if *maxBatchThreads > 128 { + *maxBatchThreads = 128 + log.Printf("[WARN] max batch threads: %d (should be between 1 and 128. recommended: 16)", *maxBatchThreads) } - if *hostnamePath == "" { - log.Fatalf("[NNTP]: Error: hostname must be set!") + if *maxBatch > 1000 { + log.Printf("[WARN] max batch: %d (should be between 100 and 1000)", *maxBatch) } // Validate command-line flag if *useShortHashLenPtr < 2 || *useShortHashLenPtr > 7 { log.Fatalf("Invalid UseShortHashLen: %d (must be between 2 and 7)", *useShortHashLenPtr) } - database.InitialBatchChannelSize = *maxBatch * *maxLoops + database.InitialBatchChannelSize = *maxBatch database.MaxBatchThreads = *maxBatchThreads database.MaxBatchSize = *maxBatch + database.MaxQueued = *maxQueued nntp.MaxReadLinesXover = int64(*maxBatch) - processor.LocalHostnamePath = *hostnamePath - processor.XoverCopy = *importOverview processor.MaxBatchSize = int64(*maxBatch) - //processor.LOOPS_PER_GROUPS = *maxLoops - - // Set global max read lines for xover - - mainConfig := config.NewDefaultConfig() - mainConfig.Server.Hostname = *hostnamePath // Initialize database (default config, data in ./data) db, err := database.OpenDatabase(nil) @@ -180,6 +158,11 @@ func main() { log.Printf("Using stored UseShortHashLen: %d", useShortHashLen) } + // Set hostname in processor with database fallback support + if err := processor.SetHostname(*nntphostname, db); err != nil { + log.Fatalf("Failed to set NNTP hostname: %v", err) + } + providers, err := db.GetProviders() if err != nil || len(providers) == 0 { // handle error appropriately @@ -216,12 +199,12 @@ func main() { pools := make([]*nntp.Pool, 0, len(providers)) for _, p := range providers { if !p.Enabled || p.Host == "" || p.Port <= 0 || p.MaxConns <= 0 { - log.Printf("Ignore disabled Provider: %s", p.Name) + //log.Printf("Ignore disabled Provider: %s", p.Name) continue } if strings.Contains(p.Host, "eternal-september") && p.MaxConns > 3 { p.MaxConns = 3 - } else if strings.Contains(p.Host, "blueworld-hosting") && p.MaxConns > 3 { + } else if strings.Contains(p.Host, "blueworldhosting") && p.MaxConns > 3 { p.MaxConns = 3 } if p.MaxConns > *maxBatch { @@ -243,6 +226,14 @@ func main() { Enabled: p.Enabled, Priority: p.Priority, MaxArtSize: p.MaxArtSize, + Posting: p.Posting, + // Copy proxy configuration from database provider + ProxyEnabled: p.ProxyEnabled, + ProxyType: p.ProxyType, + ProxyHost: p.ProxyHost, + ProxyPort: p.ProxyPort, + ProxyUsername: p.ProxyUsername, + ProxyPassword: p.ProxyPassword, } backendConfig := &nntp.BackendConfig{ @@ -256,6 +247,13 @@ func main() { //ReadTimeout: 60 * time.Second, //WriteTimeout: 30 * time.Second, Provider: configProvider, // Set the Provider field + // Copy proxy configuration from database provider + ProxyEnabled: p.ProxyEnabled, + ProxyType: p.ProxyType, + ProxyHost: p.ProxyHost, + ProxyPort: p.ProxyPort, + ProxyUsername: p.ProxyUsername, + ProxyPassword: p.ProxyPassword, } pool := nntp.NewPool(backendConfig) pool.StartCleanupWorker(5 * time.Second) @@ -266,7 +264,13 @@ func main() { } fetchDoneChan := make(chan error, 1) - shutdownChan := make(chan struct{}) // For graceful shutdown signaling + shutdownChan := make(chan struct{}) // For graceful shutdown signaling + go func() { + <-sigChan + log.Printf("[FETCHER]: Received shutdown signal, initiating graceful shutdown...") + // Signal all worker goroutines to stop + close(shutdownChan) + }() proc := processor.NewProcessor(db, pools[0], useShortHashLen) // Use first pool for import if proc == nil { log.Fatalf("[FETCHER]: Failed to create processor: %v", err) @@ -286,7 +290,16 @@ func main() { queued := 0 todo := 0 go func() { + defer close(processor.Batch.Check) for _, ng := range newsgroups { + if proc.WantShutdown(shutdownChan) { + //log.Printf("[FETCHER]: Feed Batch.Check shutdown") + return + } + if db.IsDBshutdown() { + //log.Printf("[FETCHER]: Feed Batch.Check shutdown") + return + } if wildcardNG != "" && !strings.HasPrefix(ng.Name, wildcardNG) { //log.Printf("[FETCHER] Skipping newsgroup '%s' as it does not match prefix '%s'", ng.Name, wildcardNG) continue @@ -296,17 +309,13 @@ func main() { //log.Printf("[FETCHER] ignore newsgroup '%s' err='%v' ng='%#v'", ng.Name, err, ng) continue } - if db.IsDBshutdown() { - //log.Printf("[FETCHER]: Database shutdown detected, stopping processing") - return - } + processor.Batch.Check <- &ng.Name //log.Printf("Checking ng: %s", ng.Name) mux.Lock() queued++ mux.Unlock() } - close(processor.Batch.Check) log.Printf("Queued %d newsgroups", queued) }() var wgCheck sync.WaitGroup @@ -316,6 +325,14 @@ func main() { go func(worker int, wgCheck *sync.WaitGroup, progressDB *database.ProgressDB) { defer wgCheck.Done() for ng := range processor.Batch.Check { + if proc.WantShutdown(shutdownChan) { + //log.Printf("[FETCHER]: Batch.Check shutdown") + return + } + if db.IsDBshutdown() { + //log.Printf("[FETCHER]: Batch.Check DB shutdown") + return + } groupInfo, err := proc.Pool.SelectGroup(*ng) if err != nil || groupInfo == nil { if err == nntp.ErrNewsgroupNotFound { @@ -325,6 +342,10 @@ func main() { log.Printf("[FETCHER]: Error in select ng='%s' groupInfo='%#v' err='%v'", *ng, groupInfo, err) continue } + if groupInfo.Last == 0 || groupInfo.Last < groupInfo.First { + log.Printf("[FETCHER]: Empty group '%s'", *ng) + continue + } //log.Printf("[FETCHER]: ng '%s', REMOTE groupInfo: %#v", *ng, groupInfo), var lastArticle int64 if *downloadStartDate != "" { @@ -364,7 +385,6 @@ func main() { mux.Lock() startDates[*ng] = lastArticleDate.Format("2006-01-02") mux.Unlock() - //go proc.DownloadArticlesFromDate(*ng, *lastArticleDate, 0, DLParChan, progressDB) // Use 0 for ignore threshold since group already exists } case -1: // User-requested date rescan @@ -408,8 +428,8 @@ func main() { continue } - groupInfo.First = start - groupInfo.Last = end + groupInfo.FetchStart = start + groupInfo.FetchEnd = end processor.Batch.TodoQ <- groupInfo log.Printf("[FETCHER]: TodoQ '%s' toFetch=%d start=%d end=%d", *ng, toFetch, start, end) //time.Sleep(time.Second * 2) @@ -429,6 +449,14 @@ func main() { go func(worker int) { //log.Printf("DownloadArticles: Worker %d group '%s' start", worker, groupName) for item := range processor.Batch.GetQ { + if proc.WantShutdown(shutdownChan) { + //log.Printf("[FETCHER]: Batch.GetQ shutdown") + return + } + if db.IsDBshutdown() { + //log.Printf("[FETCHER]: Batch.GetQ DB shutdown") + return + } //log.Printf("DownloadArticles: Worker %d GetArticle group '%s' article (%s)", worker, *item.GroupName, *item.MessageID) art, err := proc.Pool.GetArticle(item.MessageID, true) if err != nil || art == nil { @@ -461,14 +489,21 @@ func main() { }() for { select { - case <-shutdownChan: - //log.Printf("[FETCHER]: Worker received shutdown signal, stopping") + case _, ok := <-shutdownChan: + if !ok { + log.Printf("[FETCHER]: Worker received shutdown signal, stopping") + } return - case ng := <-processor.Batch.TodoQ: - if ng == nil { + case groupInfo := <-processor.Batch.TodoQ: + if groupInfo == nil { //log.Printf("[FETCHER]: TodoQ closed, worker stopping") return } + if proc.WantShutdown(shutdownChan) { + //log.Printf("[FETCHER]: Worker received shutdown signal, stopping") + return + } + // Check if database is shutting down if db.IsDBshutdown() { //log.Printf("[FETCHER]: TodoQ Database shutdown detected, stopping processing. still queued in TodoQ: %d", len(processor.Batch.TodoQ)) return @@ -485,7 +520,7 @@ func main() { } */ - nga, err := db.MainDBGetNewsgroup(ng.Name) + nga, err := db.MainDBGetNewsgroup(groupInfo.Name) if err != nil { log.Printf("Error in processor.Batch.TodoQ: MainDBGetNewsgroup err='%v'", err) errChan <- err @@ -496,111 +531,107 @@ func main() { //log.Printf("[FETCHER]: start *importOverview=%t '%s' (%d-%d) [%d/%d|Q:%d] --- ", *importOverview, ng.Name, ng.First, ng.Last, todo, queued, len(processor.Batch.TodoQ)) mux.Unlock() // Import articles for the selected group - switch *importOverview { - case false: - //log.Printf("[FETCHER]: Downloading articles for newsgroup: %s", ng.Name) - // Check if date-based downloading is requested - var useStartDate string - mux.Lock() - if startDates[ng.Name] != "" { - useStartDate = startDates[ng.Name] - } else if *downloadStartDate != "" { - useStartDate = *downloadStartDate - } - mux.Unlock() - if useStartDate != "" { - startDate, err := time.Parse("2006-01-02", useStartDate) - if err != nil { - log.Fatalf("[FETCHER]: Invalid start date format '%s': %v (expected YYYY-MM-DD)", useStartDate, err) - } - log.Printf("[FETCHER]: Starting ng: '%s' from date: %s", ng.Name, startDate.Format("2006-01-02")) - //time.Sleep(3 * time.Second) // debug sleep - err = proc.DownloadArticlesFromDate(ng.Name, startDate, *ignoreInitialTinyGroups, DLParChan, progressDB) - if err != nil { - log.Printf("[FETCHER]: DownloadArticlesFromDate5 failed: %v", err) - errChan <- err - continue - } - } else if nga.ExpiryDays > 0 { - // Check if group already has articles to decide between initial vs incremental download - // Use optimized main database check instead of opening group database - articleCount, err := db.GetArticleCountFromMainDB(ng.Name) - if err != nil { - log.Printf("[FETCHER]: Failed to get article count from main DB for '%s': %v", ng.Name, err) - errChan <- err - continue - } - if articleCount == 0 { - // Initial download: use expiry_days to avoid downloading old articles - startDate := time.Now().AddDate(0, 0, -nga.ExpiryDays) - log.Printf("[FETCHER]: Initial download for group with expiry_days=%d, starting from calculated date: %s", nga.ExpiryDays, startDate.Format("2006-01-02")) - //time.Sleep(3 * time.Second) // debug sleep - err = proc.DownloadArticlesFromDate(ng.Name, startDate, *ignoreInitialTinyGroups, DLParChan, progressDB) + //log.Printf("[FETCHER]: Downloading articles for newsgroup: %s", ng.Name) + // Check if date-based downloading is requested + var useStartDate string + mux.Lock() + if startDates[groupInfo.Name] != "" { + useStartDate = startDates[groupInfo.Name] + } else if *downloadStartDate != "" { + useStartDate = *downloadStartDate + } + mux.Unlock() + if useStartDate != "" { + startDate, err := time.Parse("2006-01-02", useStartDate) + if err != nil { + log.Fatalf("[FETCHER]: Invalid start date format '%s': %v (expected YYYY-MM-DD)", useStartDate, err) + } + log.Printf("[FETCHER]: Starting ng: '%s' from date: %s", groupInfo.Name, startDate.Format("2006-01-02")) + //time.Sleep(3 * time.Second) // debug sleep + err = proc.DownloadArticlesFromDate(groupInfo.Name, startDate, DLParChan, progressDB, groupInfo, shutdownChan) + if err != nil { + if err == processor.ErrIsEmptyGroup { + err = progressDB.UpdateProgress(proc.Pool.Backend.Provider.Name, groupInfo.Name, 0) if err != nil { - errChan <- err - log.Printf("[FETCHER]: DownloadArticlesFromDate6 failed: %v", err) continue } - } else { - // Incremental download: continue from where we left off - log.Printf("[FETCHER]: Incremental download for newsgroup: '%s' (has %d existing articles)", ng.Name, articleCount) - //time.Sleep(3 * time.Second) // debug sleep - err = proc.DownloadArticles(ng.Name, *ignoreInitialTinyGroups, DLParChan, progressDB, ng.First, ng.Last) - if err != nil { - log.Printf("[FETCHER]: DownloadArticles7 failed: %v", err) - errChan <- err + errChan <- nil + continue + } + log.Printf("[FETCHER]: DownloadArticlesFromDate5 failed: %v", err) + errChan <- err + continue + } + } else if nga.ExpiryDays > 0 { + // Check if group already has articles to decide between initial vs incremental download + // Use optimized main database check instead of opening group database + articleCount, err := db.GetArticleCountFromMainDB(groupInfo.Name) + if err != nil { + log.Printf("[FETCHER]: Failed to get article count from main DB for '%s': %v", groupInfo.Name, err) + errChan <- err + continue + } + if articleCount == 0 { + // Initial download: use expiry_days to avoid downloading old articles + startDate := time.Now().AddDate(0, 0, -nga.ExpiryDays) + log.Printf("[FETCHER]: Initial download for group with expiry_days=%d, starting from calculated date: %s", nga.ExpiryDays, startDate.Format("2006-01-02")) + //time.Sleep(3 * time.Second) // debug sleep + err = proc.DownloadArticlesFromDate(groupInfo.Name, startDate, DLParChan, progressDB, groupInfo, shutdownChan) + if err != nil { + if err == processor.ErrIsEmptyGroup { + err = progressDB.UpdateProgress(proc.Pool.Backend.Provider.Name, groupInfo.Name, 0) + if err != nil { + continue + } + errChan <- nil continue } + errChan <- err + log.Printf("[FETCHER]: DownloadArticlesFromDate6 failed: %v", err) + continue } } else { - log.Printf("[FETCHER]: Downloading articles for newsgroup: '%s' (%d - %d) (no expiry limit)", ng.Name, ng.First, ng.Last) + // Incremental download: continue from where we left off + log.Printf("[FETCHER]: Incremental download for newsgroup: '%s' (has %d existing articles)", groupInfo.Name, articleCount) //time.Sleep(3 * time.Second) // debug sleep - err = proc.DownloadArticles(ng.Name, *ignoreInitialTinyGroups, DLParChan, progressDB, ng.First, ng.Last) + err = proc.DownloadArticles(groupInfo.Name, DLParChan, progressDB, groupInfo.FetchStart, groupInfo.FetchEnd, shutdownChan) if err != nil { - if err != processor.ErrUpToDate { - log.Printf("[FETCHER]: DownloadArticles8 failed: %v", err) - } + log.Printf("[FETCHER]: DownloadArticles7 failed: %v", err) errChan <- err continue } } + } else { + log.Printf("[FETCHER]: Downloading articles for newsgroup: '%s' (%d - %d) (no expiry limit)", groupInfo.Name, groupInfo.FetchStart, groupInfo.FetchEnd) + //time.Sleep(3 * time.Second) // debug sleep + err = proc.DownloadArticles(groupInfo.Name, DLParChan, progressDB, groupInfo.FetchStart, groupInfo.FetchEnd, shutdownChan) if err != nil { if err != processor.ErrUpToDate { - log.Printf("DownloadArticles9 failed: %v", err) + log.Printf("[FETCHER]: DownloadArticles8 failed: %v", err) } + errChan <- err continue } - //}(DLParChan) // end go func - /* - case true: - log.Printf("[FETCHER]: Experimental! Start DownloadArticlesViaOverview for group '%s'", ng.Name) - err = proc.DownloadArticlesViaOverview(ng.Name) - if err != nil { - log.Printf("[FETCHER]: DownloadArticlesViaOverview failed: %v", err) - continue - } - fmt.Println("[FETCHER]: ✓ Article import complete.") - - groupDBs, err := db.GetGroupDBs(ng.Name) - if err != nil { - log.Fatalf("[FETCHER]: Failed to get group DBs for '%s': %v", ng.Name, err) - } - defer groupDBs.Return(db) - */ + } + if err != nil { + if err != processor.ErrUpToDate { + log.Printf("DownloadArticles9 failed: %v", err) + } + continue } } } }(&waitHere) } - db.WG.Done() // backwards compat... TODO remove this + db.WG.Done() // Wait for either shutdown signal or server error select { - case <-sigChan: - log.Printf("[FETCHER]: Received shutdown signal, initiating graceful shutdown...") - // Signal all worker goroutines to stop - close(shutdownChan) + case _, ok := <-shutdownChan: + if !ok { + //log.Printf("[FETCHER]: Shutdown channel closed, initiating graceful shutdown...") + } case err := <-fetchDoneChan: log.Printf("[FETCHER]: DONE! err='%v'", err) } @@ -632,306 +663,6 @@ func main() { log.Printf("[FETCHER]: Graceful shutdown completed. Exiting here.") } -func ConnectionTest(host *string, port *int, username *string, password *string, ssl *bool, timeout *int, fetchNewsgroup string, testMsg *string) error { - // Create a test provider config - testProvider := &config.Provider{ - Name: "test", - Host: *host, - Port: *port, - SSL: *ssl, - Username: *username, - Password: *password, - MaxConns: 3, - Enabled: true, - Priority: 1, - } - - // Create Test client configuration - backenConfig := &nntp.BackendConfig{ - Host: *host, - Port: *port, - SSL: *ssl, - Username: *username, - Password: *password, - MaxConns: 3, // Default max connections - Provider: testProvider, // Set the Provider field - ConnectTimeout: time.Duration(*timeout) * time.Second, - } - - fmt.Printf("Testing NNTP connection to %s:%d (SSL: %v)\n", *host, *port, *ssl) - if *username != "" { - fmt.Printf("Authentication: %s\n", *username) - } else { - fmt.Println("Authentication: None") - } - - // Test 1: Basic connection only use this in a test! - // Proper way is #2 to use the connection pool below! - fmt.Println("\n=== Test 1: Test Basic Connection without backend Counter! ===") - client := nntp.NewConn(backenConfig) - start := time.Now() - err := client.Connect() - if err != nil { - log.Fatalf("Failed to connect: %v", err) - } - fmt.Printf("✓ Connection successful (took %v)\n", time.Since(start)) - client.CloseFromPoolOnly() // only use this in a test! - - // Test 2: Connection pool - fmt.Println("\n=== Test 2: Connection Pool ===") - pool := nntp.NewPool(backenConfig) - defer pool.ClosePool() - - pool.StartCleanupWorker(5 * time.Second) - - poolClient, err := pool.Get() - if err != nil { - log.Fatalf("Failed to get connection from pool: %v", err) - } - - fmt.Printf("✓ Pool connection successful\n") - - stats := pool.Stats() - fmt.Printf("Pool Stats: Max=%d, Active=%d, Idle=%d, Created=%d\n", - stats.MaxConnections, stats.ActiveConnections, stats.IdleConnections, stats.TotalCreated) - poolClient.Pool.Put(poolClient) // Return connection to pool - - // Test 3: List groups (first 10) - fmt.Println("\n=== Test 3: List Groups ===") - poolClient, err = pool.Get() // Get a connection from the pool - if err != nil { - log.Fatalf("Failed to get connection from pool: %v", err) - } - var groups []nntp.GroupInfo - func() { - defer poolClient.Pool.Put(poolClient) // Ensure connection is always returned - var err error - groups, err = poolClient.ListGroups() - if err != nil { - fmt.Printf("⚠ Failed to list groups: %v\n", err) - } else { - fmt.Printf("✓ Retrieved %d groups\n", len(groups)) - - // Show first 10 groups - limit := 10 - if len(groups) < limit { - limit = len(groups) - } - - fmt.Println("First groups:") - for i := 0; i < limit; i++ { - group := groups[i] - fmt.Printf(" %s: %d articles (%d-%d) posting=%v\n", - group.Name, group.Count, group.First, group.Last, group.PostingOK) - } - } - }() - - // Test 4: Select a specific group (or try first available) - poolClient, err = pool.Get() // Get a connection from the pool - if err != nil { - log.Fatalf("Failed to get connection from pool: %v", err) - } - func() { - defer poolClient.Pool.Put(poolClient) // Ensure connection is always returned - if fetchNewsgroup != "" { - fmt.Printf("\n=== Test 4: Select Group '%s' ===\n", fetchNewsgroup) - groupInfo, _, err := poolClient.SelectGroup(fetchNewsgroup) - if err != nil { - fmt.Printf("⚠ Failed to select group: %v\n", err) - } else { - fmt.Printf("✓ Group selected: %s\n", groupInfo.Name) - fmt.Printf(" Articles: %d (%d-%d)\n", groupInfo.Count, groupInfo.First, groupInfo.Last) - fmt.Printf(" Posting: %v\n", groupInfo.PostingOK) - } - } else if len(groups) > 0 { - // Try to select the first few groups, skipping problematic ones - fmt.Println("\n=== Test 4: Auto-select Available Group ===") - for i, group := range groups { - if i >= 5 { // Try max 5 groups - break - } - - // Skip known problematic groups - if group.Name == "control" || group.Name == "junk" { - fmt.Printf("Skipping problematic group: %s\n", group.Name) - continue - } - - fmt.Printf("Trying to select group: %s\n", group.Name) - groupInfo, _, err := poolClient.SelectGroup(group.Name) - if err != nil { - fmt.Printf("⚠ Failed to select group %s: %v (trying next)\n", group.Name, err) - continue - } - - fmt.Printf("✓ Successfully selected group: %s\n", groupInfo.Name) - fmt.Printf(" Articles: %d (%d-%d)\n", groupInfo.Count, groupInfo.First, groupInfo.Last) - fmt.Printf(" Posting: %v\n", groupInfo.PostingOK) - break - } - } - }() - - // Test 5: Test specific message ID - if *testMsg != "" { - poolClient, err = pool.Get() // Get a connection from the pool - if err != nil { - log.Fatalf("Test 5 Failed to get connection from pool: %v", err) - } - func() { - defer poolClient.Pool.Put(poolClient) // Ensure connection is always returned - fmt.Printf("\n=== Test 5: Test Message ID '%s' ===\n", *testMsg) - - // Test STAT command - exists, err := poolClient.StatArticle(*testMsg) - if err != nil { - fmt.Printf("⚠ STAT failed: %v\n", err) - } else { - fmt.Printf("✓ STAT result: exists=%v\n", exists) - } - - if exists { - // Test HEAD command - article, err := poolClient.GetHead(*testMsg) - if err != nil { - fmt.Printf("⚠ HEAD failed: %v\n", err) - } else { - fmt.Printf("✓ HEAD successful, %d headers\n", len(article.Headers)) - - // Show some key headers - if subject := article.Headers["subject"]; len(subject) > 0 { - fmt.Printf(" Subject: %s\n", subject[0]) - } - if from := article.Headers["from"]; len(from) > 0 { - fmt.Printf(" From: %s\n", from[0]) - } - if date := article.Headers["date"]; len(date) > 0 { - fmt.Printf(" Date: %s\n", date[0]) - } - } - } - }() - } - - // Test 6: XOVER (Overview data) - if fetchNewsgroup != "" { - poolClient, err = pool.Get() // Get a connection from the pool - if err != nil { - log.Fatalf("Test 6 Failed to get connection from pool: %v", err) - } - func() { - defer poolClient.Pool.Put(poolClient) // Ensure connection is always returned - fmt.Printf("\n=== Test 6: XOVER for group '%s' ===\n", fetchNewsgroup) - groupInfo, _, err := poolClient.SelectGroup(fetchNewsgroup) - if err != nil { - fmt.Printf("⚠ Failed to select group for XOVER: %v\n", err) - } else { - // Get overview data for first 10 articles - start := groupInfo.First - end := start + 9 - if end > groupInfo.Last { - end = groupInfo.Last - } - enforceLimit := false - fmt.Printf("Getting XOVER data for articles %d-%d...\n", start, end) - overviews, err := poolClient.XOver(fetchNewsgroup, start, end, enforceLimit) - if err != nil { - fmt.Printf("⚠ XOVER failed: %v\n", err) - } else { - fmt.Printf("✓ Retrieved %d overview records\n", len(overviews)) - for i, ov := range overviews { - if i >= 3 { // Show only first 3 - break - } - fmt.Printf(" Article %d: %s (from: %s, %d bytes)\n", - ov.ArticleNum, ov.Subject[:min(50, len(ov.Subject))], - ov.From[:min(30, len(ov.From))], ov.Bytes) - } - } - } - }() - } - - // Test 7: XHDR (Header field extraction) - if fetchNewsgroup != "" { - poolClient, err = pool.Get() // Get a connection from the pool - if err != nil { - log.Fatalf("Test 7 Failed to get connection from pool: %v", err) - } - func() { - defer poolClient.Pool.Put(poolClient) // Ensure connection is always returned - fmt.Printf("\n=== Test 7: XHDR for group '%s' ===\n", fetchNewsgroup) - groupInfo, _, err := poolClient.SelectGroup(fetchNewsgroup) - if err != nil { - fmt.Printf("⚠ Failed to select group for XHDR: %v\n", err) - } else { - // Get subject headers for first 5 articles - start := groupInfo.First - end := start + 4 - if end > groupInfo.Last { - end = groupInfo.Last - } - - fmt.Printf("Getting XHDR Subject for articles %d-%d...\n", start, end) - headers, err := poolClient.XHdr(fetchNewsgroup, "Subject", start, end) - if err != nil { - fmt.Printf("⚠ XHDR failed: %v\n", err) - } else { - fmt.Printf("✓ Retrieved %d subject headers\n", len(headers)) - for i, hdr := range headers { - if i >= 3 { // Show only first 3 - break - } - fmt.Printf(" Article %d: %s\n", hdr.ArticleNum, - hdr.Value[:min(60, len(hdr.Value))]) - } - } - } - }() - } - - // Test 8: LISTGROUP (Article numbers) - if fetchNewsgroup != "" && !strings.Contains(fetchNewsgroup, "*") && !strings.Contains(fetchNewsgroup, "$") { - poolClient, err = pool.Get() // Get a connection from the pool - if err != nil { - log.Fatalf("Test 8 Failed to get connection from pool: %v", err) - } - func() { - defer poolClient.Pool.Put(poolClient) // Ensure connection is always returned - fmt.Printf("\n=== Test 8: LISTGROUP for '%s' ===\n", fetchNewsgroup) - // Get first 20 article numbers - fmt.Printf("Getting article numbers (limited)...\n") - articleNums, err := poolClient.ListGroup(fetchNewsgroup, 0, 0) // Get all (limited by server) - if err != nil { - fmt.Printf("⚠ LISTGROUP failed: %v\n", err) - } else { - fmt.Printf("✓ Retrieved %d article numbers\n", len(articleNums)) - if len(articleNums) > 0 { - fmt.Printf(" First articles: ") - for i, num := range articleNums { - if i >= 10 { // Show first 10 - fmt.Printf("...") - break - } - fmt.Printf("%d ", num) - } - fmt.Printf("\n Last articles: ") - start := len(articleNums) - 5 - if start < 0 { - start = 0 - } - for i := start; i < len(articleNums); i++ { - fmt.Printf("%d ", articleNums[i]) - } - fmt.Println() - } - } - }() - } - return err -} // end func ConnectionTest - // getRealMemoryUsage gets actual RSS memory usage from /proc/self/status on Linux func getRealMemoryUsage() (uint64, error) { file, err := os.Open("/proc/self/status") @@ -960,7 +691,7 @@ func getRealMemoryUsage() (uint64, error) { // UpdateNewsgroupList fetches the remote newsgroup list from the first enabled provider // and adds all groups to the database that we don't already have -func UpdateNewsgroupList(host *string, port *int, username *string, password *string, ssl *bool, timeout *int, updateList *string) error { +func UpdateNewsgroupList(updateList *string, updateListForce bool) error { log.Printf("Starting newsgroup list update from remote server...") // Initialize database @@ -1029,13 +760,13 @@ func UpdateNewsgroupList(host *string, port *int, username *string, password *st if updatePattern == "$all" { addAllGroups = true - log.Printf("Adding all newsgroups from remote server") + log.Printf("Listing all newsgroups from remote server") } else if strings.HasSuffix(updatePattern, "*") { groupPrefix = strings.TrimSuffix(updatePattern, "*") - log.Printf("Adding newsgroups with prefix: '%s'", groupPrefix) + log.Printf("Listing newsgroups with prefix: '%s'", groupPrefix) } else if updatePattern != "" { groupPrefix = updatePattern - log.Printf("Adding newsgroups matching: '%s'", groupPrefix) + log.Printf("Listing newsgroups matching: '%s'", groupPrefix) } else { return fmt.Errorf("invalid update pattern: '%s' (use 'group.*' or '$all')", updatePattern) } @@ -1052,11 +783,12 @@ func UpdateNewsgroupList(host *string, port *int, username *string, password *st existingGroups[group.Name] = true } - log.Printf("Found %d existing newsgroups in local database", len(localGroups)) + log.Printf("Found %d newsgroups in local database", len(localGroups)) // Add new newsgroups that don't exist locally and match the pattern newGroupCount := 0 skippedCount := 0 + var messages int64 for _, remoteGroup := range remoteGroups { // Apply prefix filtering if !addAllGroups { @@ -1075,20 +807,25 @@ func UpdateNewsgroupList(host *string, port *int, username *string, password *st CreatedAt: time.Now().UTC(), // Default created at } - // Insert the new newsgroup - err := db.InsertNewsgroup(newGroup) - if err != nil { - log.Printf("Failed to insert newsgroup '%s': %v", remoteGroup.Name, err) - continue - } + if updateListForce { + // Insert the new newsgroup + err := db.InsertNewsgroup(newGroup) + if err != nil { + log.Printf("Failed to insert newsgroup '%s': %v", remoteGroup.Name, err) + continue + } - log.Printf("Added new newsgroup: %s", remoteGroup.Name) + log.Printf("Added new newsgroup: %s", remoteGroup.Name) + } else { + log.Printf("New newsgroup: %s (not added) lo=%d hi=%d messages=%d", remoteGroup.Name, remoteGroup.First, remoteGroup.Last, remoteGroup.Count) + } newGroupCount++ + messages += remoteGroup.Count } } - log.Printf("Newsgroup list update completed: %d new groups added, %d skipped (prefix filter), out of %d remote groups", - newGroupCount, skippedCount, len(remoteGroups)) + log.Printf("Newsgroup list update completed: %d new groups added, %d skipped (prefix filter), out of %d remote groups with total: %d messages", + newGroupCount, skippedCount, len(remoteGroups), messages) return nil } diff --git a/cmd/nntp-server/main.go b/cmd/nntp-server/main.go index 10fb7772..10ff3ac6 100644 --- a/cmd/nntp-server/main.go +++ b/cmd/nntp-server/main.go @@ -15,7 +15,7 @@ import ( ) var ( - hostnamePath string + nntphostname string nntptcpport int nntptlsport int nntpcertFile string @@ -29,11 +29,8 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion log.Printf("Starting go-pugleaf dedicated NNTP Server (version: %s)", config.AppVersion) - // Example configuration - mainConfig := config.NewDefaultConfig() - log.Printf("Starting go-pugleaf dedicated NNTP Server (version: %s)", appVersion) - flag.StringVar(&hostnamePath, "nntphostname", "", "Your hostname must be set!") + flag.StringVar(&nntphostname, "nntphostname", "", "Your hostname must be set!") flag.IntVar(&nntptcpport, "nntptcpport", 0, "NNTP TCP port") flag.IntVar(&nntptlsport, "nntptlsport", 0, "NNTP TLS port") flag.StringVar(&nntpcertFile, "nntpcertfile", "", "NNTP TLS certificate file (/path/to/fullchain.pem)") @@ -42,6 +39,7 @@ func main() { flag.IntVar(&maxConnections, "maxconnections", 500, "allow max of N authenticated connections (default: 500)") flag.Parse() + mainConfig := config.NewDefaultConfig() mainConfig.Server.NNTP.Enabled = true // Override config with command-line flags if provided if nntptcpport > 0 { @@ -62,17 +60,14 @@ func main() { log.Printf("[NNTP]: No NNTP TLS port flag provided") } - if hostnamePath == "" { - log.Fatalf("[NNTP]: Error: hostname must be set!") - } + // Note: hostname can be empty here since SetHostname will check database for fallback if maxConnections <= 0 { log.Fatalf("[NNTP]: Error: max connections must be greater than 0") } if maxConnections > 500 { // Default is 500, but allow higher if specified log.Printf("[NNTP]: WARNING! Setting max connections to %d: You may hit filedescriptor limits! rise ulimit -n to maxConnections * 2 !", maxConnections) } - mainConfig.Server.Hostname = hostnamePath - processor.LocalHostnamePath = hostnamePath + mainConfig.Server.Hostname = nntphostname mainConfig.Server.NNTP.MaxConns = maxConnections log.Printf("[NNTP]: Using NNTP configuration %#v", mainConfig.Server.NNTP) @@ -93,6 +88,11 @@ func main() { log.Fatalf("Failed to apply database migrations: %v", err) } + // Set hostname in processor with database fallback support + if err := processor.SetHostname(nntphostname, db); err != nil { + log.Fatalf("[NNTP]: Failed to set NNTP hostname: %v", err) + } + // Validate command-line flag if useShortHashLen < 2 || useShortHashLen > 7 { log.Fatalf("Invalid UseShortHashLen: %d (must be between 2 and 7)", useShortHashLen) diff --git a/cmd/nntp-transfer/main.go b/cmd/nntp-transfer/main.go new file mode 100644 index 00000000..d4a1562f --- /dev/null +++ b/cmd/nntp-transfer/main.go @@ -0,0 +1,947 @@ +// NNTP article transfer tool for go-pugleaf +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "time" + + "github.com/go-while/go-pugleaf/internal/config" + "github.com/go-while/go-pugleaf/internal/database" + "github.com/go-while/go-pugleaf/internal/models" + "github.com/go-while/go-pugleaf/internal/nntp" + "github.com/go-while/go-pugleaf/internal/processor" +) + +var dbBatchSize int64 = 1000 // Load 1000 articles from DB at a time + +// showUsageExamples displays usage examples for NNTP transfer +func showUsageExamples() { + fmt.Println("\n=== NNTP Transfer Tool - Usage Examples ===") + fmt.Println("The NNTP transfer tool sends articles via CHECK/TAKETHIS commands.") + fmt.Println() + fmt.Println("Connection Configuration:") + fmt.Println(" ./nntp-transfer -host news.server.local -group news.admin.*") + fmt.Println(" ./nntp-transfer -host news.server.local -username user -password pass -group alt.test") + fmt.Println(" ./nntp-transfer -host news.server.local -port 119 -ssl=false -group alt.test") + fmt.Println() + fmt.Println("Proxy Configuration:") + fmt.Println(" ./nntp-transfer -host news.server.local -socks5 127.0.0.1:9050 -group alt.test") + fmt.Println(" ./nntp-transfer -host news.server.local -socks4 proxy.example.com:1080 -group alt.test") + fmt.Println(" ./nntp-transfer -host news.server.local -socks5 proxy.example.com:1080 -proxy-username user -proxy-password pass -group alt.test") + fmt.Println() + fmt.Println("Performance Tuning:") + fmt.Println(" ./nntp-transfer -host news.server.local -max-threads 4 -group alt.*") + fmt.Println() + fmt.Println("Date Filtering:") + fmt.Println(" ./nntp-transfer -host news.server.local -date-beg 2024-01-01 -group alt.test") + fmt.Println(" ./nntp-transfer -host news.server.local -date-end 2024-12-31 -group alt.test") + fmt.Println(" ./nntp-transfer -host news.server.local -date-beg 2024-01-01T00:00:00 -date-end 2024-01-31T23:59:59 -group alt.test") + fmt.Println() + fmt.Println("Dry Run Mode:") + fmt.Println(" ./nntp-transfer -host news.server.local -dry-run -group alt.test") + fmt.Println() + + fmt.Println("Show ALL command line flags:") + fmt.Println(" ./nntp-transfer -h") + fmt.Println() +} + +var appVersion = "-unset-" + +func main() { + config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches + log.Printf("Starting go-pugleaf NNTP Transfer Tool (version %s)", config.AppVersion) + + // Command line flags for NNTP transfer configuration + var ( + // Required flags + transferGroup = flag.String("group", "", "Newsgroup to transfer (supports wildcards like alt.* or news.admin.*)") + + // Connection configuration + host = flag.String("host", "", "Target NNTP hostname") + port = flag.Int("port", 563, "Target NNTP port (common: 119 -ssl=false OR 563 -ssl=true)") + username = flag.String("username", "", "Target NNTP username") + password = flag.String("password", "", "Target NNTP password") + ssl = flag.Bool("ssl", true, "Use SSL/TLS connection") + timeout = flag.Int("timeout", 30, "Connection timeout in seconds") + + // Proxy configuration + proxySocks4 = flag.String("socks4", "", "SOCKS4 proxy address (host:port)") + proxySocks5 = flag.String("socks5", "", "SOCKS5 proxy address (host:port)") + proxyUsername = flag.String("proxy-username", "", "Proxy authentication username") + proxyPassword = flag.String("proxy-password", "", "Proxy authentication password") + + // Transfer configuration + batchCheck = flag.Int("batch-check", 25, "Number of message-IDs to send in a single CHECK command") + batchDB = flag.Int64("batch-db", 1000, "Fetch N articles from DB in a batch") + maxThreads = flag.Int("max-threads", 1, "Transfer N newsgroups in concurrent threads. Each thread uses 1 connection.") + + // Operation options + dryRun = flag.Bool("dry-run", false, "Show what would be transferred without actually sending") + testConn = flag.Bool("test-conn", false, "Test connection and exit") + showHelp = flag.Bool("help", false, "Show usage examples and exit") + + // Date filtering options + startDate = flag.String("date-beg", "", "Start date for article transfer (format: 2006-01-02 [YYYY-MM-DD] or 2006-01-02T15:04:05)") + endDate = flag.String("date-end", "", "End date for article transfer (format: 2006-01-02 [YYYY-MM-DD] or 2006-01-02T15:04:05)") + + // History configuration + useShortHashLen = flag.Int("useshorthashlen", 7, "Short hash length for history storage (2-7, default: 7)") + ) + flag.Parse() + + // Show help if requested + if *showHelp { + showUsageExamples() + os.Exit(0) + } + + if *transferGroup == "" { + log.Fatalf("Error: -group must be set!") + } + + // Validate batch size + if *batchCheck < 1 || *batchCheck > 100 { + log.Fatalf("Error: batch-check must be between 1 and 100 (got %d)", *batchCheck) + } + + // Validate batch size + if *batchDB < 100 { + *batchDB = 100 + } + dbBatchSize = *batchDB + + // Validate thread count + if *maxThreads < 1 || *maxThreads > 500 { + log.Fatalf("Error: max-threads must be between 1 and 500 (got %d)", *maxThreads) + } + + // Validate UseShortHashLen + if *useShortHashLen < 2 || *useShortHashLen > 7 { + log.Fatalf("Invalid UseShortHashLen: %d (must be between 2 and 7)", *useShortHashLen) + } + + // Parse and validate date filters + var startTime, endTime *time.Time + if *startDate != "" { + parsed, err := parseDateTime(*startDate) + if err != nil { + log.Fatalf("Invalid start-date format: %v. Use format: 2006-01-02 or 2006-01-02T15:04:05", err) + } + startTime = &parsed + log.Printf("Filtering articles from: %s", startTime.Format("2006-01-02 15:04:05")) + } + if *endDate != "" { + parsed, err := parseDateTime(*endDate) + if err != nil { + log.Fatalf("Invalid end-date format: %v. Use format: 2006-01-02 or 2006-01-02T15:04:05", err) + } + endTime = &parsed + log.Printf("Filtering articles to: %s", endTime.Format("2006-01-02 15:04:05")) + } + if startTime != nil && endTime != nil && startTime.After(*endTime) { + log.Fatalf("Start date (%s) cannot be after end date (%s)", startTime.Format("2006-01-02"), endTime.Format("2006-01-02")) + } + + // Parse and validate proxy configuration + var proxyConfig *ProxyConfig + if *proxySocks4 != "" && *proxySocks5 != "" { + log.Fatalf("Cannot specify both SOCKS4 and SOCKS5 proxy") + } + if *proxySocks4 != "" { + config, err := parseProxyConfig(*proxySocks4, "socks4", *proxyUsername, *proxyPassword) + if err != nil { + log.Fatalf("Invalid SOCKS4 proxy configuration: %v", err) + } + proxyConfig = config + log.Printf("Using SOCKS4 proxy: %s:%d", proxyConfig.Host, proxyConfig.Port) + } + if *proxySocks5 != "" { + config, err := parseProxyConfig(*proxySocks5, "socks5", *proxyUsername, *proxyPassword) + if err != nil { + log.Fatalf("Invalid SOCKS5 proxy configuration: %v", err) + } + proxyConfig = config + log.Printf("Using SOCKS5 proxy: %s:%d", proxyConfig.Host, proxyConfig.Port) + } + + // Test connection if requested + if *testConn { + if err := testConnection(host, port, username, password, ssl, timeout, proxyConfig); err != nil { + log.Fatalf("Connection test failed: %v", err) + } + log.Printf("Connection test successful!") + os.Exit(0) + } + + // Initialize database (default config, data in ./data) + db, err := database.OpenDatabase(nil) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + + // Set up cross-platform signal handling for graceful shutdown + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt) // Cross-platform (Ctrl+C on both Windows and Linux) + + db.WG.Add(1) // Add for this transfer process + + // Get UseShortHashLen from database (with safety check) + storedUseShortHashLen, isLocked, err := db.GetHistoryUseShortHashLen(*useShortHashLen) + if err != nil { + log.Fatalf("Failed to get UseShortHashLen from database: %v", err) + } + var finalUseShortHashLen int + if !isLocked { + // First run: store the provided value + finalUseShortHashLen = *useShortHashLen + err = db.SetHistoryUseShortHashLen(finalUseShortHashLen) + if err != nil { + log.Fatalf("Failed to store UseShortHashLen in database: %v", err) + } + log.Printf("First run: UseShortHashLen set to %d and stored in database", finalUseShortHashLen) + } else { + // Subsequent runs: use stored value and warn if different + finalUseShortHashLen = storedUseShortHashLen + if *useShortHashLen != finalUseShortHashLen { + log.Printf("WARNING: Command-line UseShortHashLen (%d) differs from stored value (%d). Using stored value to prevent data corruption.", *useShortHashLen, finalUseShortHashLen) + } + log.Printf("Using stored UseShortHashLen: %d", finalUseShortHashLen) + } + + // Create target server connection pool + targetProvider := &config.Provider{ + Name: "transfer:" + *host, + Host: *host, + Port: *port, + SSL: *ssl, + Username: *username, + Password: *password, + MaxConns: *maxThreads, + Enabled: true, + Priority: 1, + MaxArtSize: 0, // No size limit for transfers + } + + backendConfig := &nntp.BackendConfig{ + Host: *host, + Port: *port, + SSL: *ssl, + Username: *username, + Password: *password, + MaxConns: *maxThreads, + Provider: targetProvider, + ConnectTimeout: time.Duration(*timeout) * time.Second, + } + + // Apply proxy configuration if specified + if proxyConfig != nil { + backendConfig.ProxyEnabled = proxyConfig.Enabled + backendConfig.ProxyType = proxyConfig.Type + backendConfig.ProxyHost = proxyConfig.Host + backendConfig.ProxyPort = proxyConfig.Port + backendConfig.ProxyUsername = proxyConfig.Username + backendConfig.ProxyPassword = proxyConfig.Password + } + + pool := nntp.NewPool(backendConfig) + pool.StartCleanupWorker(5 * time.Second) + defer pool.ClosePool() + + log.Printf("Created connection pool for target server '%s:%d' with max %d connections", *host, *port, *maxThreads) + + // Get newsgroups to transfer + newsgroups, err := getNewsgroupsToTransfer(db, *transferGroup) + if err != nil { + log.Fatalf("Failed to get newsgroups: %v", err) + } + + if len(newsgroups) == 0 { + log.Printf("No newsgroups found matching pattern: %s", *transferGroup) + os.Exit(0) + } + + log.Printf("Found %d newsgroups to transfer", len(newsgroups)) + + // Initialize processor for article handling + proc := processor.NewProcessor(db, pool, finalUseShortHashLen) + if proc == nil { + log.Fatalf("Failed to create processor") + } + + // Set up shutdown handling + shutdownChan := make(chan struct{}) + transferDoneChan := make(chan error, 1) + + // Start transfer process + go func() { + transferDoneChan <- runTransfer(db, proc, pool, newsgroups, *batchCheck, *maxThreads, *dryRun, startTime, endTime, shutdownChan) + }() + + // Wait for either shutdown signal or transfer completion + select { + case <-sigChan: + log.Printf("Received shutdown signal, initiating graceful shutdown...") + close(shutdownChan) + case err := <-transferDoneChan: + if err != nil { + log.Printf("Transfer completed with error: %v", err) + } else { + log.Printf("Transfer completed successfully") + } + } + + // Close processor + if proc != nil { + if err := proc.Close(); err != nil { + log.Printf("Warning: Failed to close processor: %v", err) + } else { + log.Printf("Processor closed successfully") + } + } + + // Wait for database operations to complete + db.WG.Done() + db.WG.Wait() + + // Shutdown database + if err := db.Shutdown(); err != nil { + log.Printf("Failed to shutdown database: %v", err) + os.Exit(1) + } else { + log.Printf("Database shutdown successfully") + } + + log.Printf("Graceful shutdown completed. Exiting.") +} + +// parseDateTime parses a date string in multiple supported formats +func parseDateTime(dateStr string) (time.Time, error) { + // Try different date formats + formats := []string{ + "2006-01-02", // YYYY-MM-DD + "2006-01-02T15:04:05", // YYYY-MM-DDTHH:MM:SS + "2006-01-02 15:04:05", // YYYY-MM-DD HH:MM:SS + "2006-01-02T15:04:05Z", // YYYY-MM-DDTHH:MM:SSZ + } + + for _, format := range formats { + if parsed, err := time.Parse(format, dateStr); err == nil { + return parsed, nil + } + } + + return time.Time{}, fmt.Errorf("unsupported date format: %s", dateStr) +} + +// ProxyConfig holds proxy configuration parsed from command line flags +type ProxyConfig struct { + Enabled bool + Type string // "socks4" or "socks5" + Host string + Port int + Username string + Password string +} + +// parseProxyConfig parses proxy address (host:port) and creates proxy configuration +func parseProxyConfig(address, proxyType, username, password string) (*ProxyConfig, error) { + if address == "" { + return nil, fmt.Errorf("proxy address cannot be empty") + } + + // Parse host:port + parts := strings.Split(address, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("proxy address must be in format host:port, got: %s", address) + } + + host := parts[0] + if host == "" { + return nil, fmt.Errorf("proxy host cannot be empty") + } + + port, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid proxy port: %s", parts[1]) + } + if port <= 0 || port > 65535 { + return nil, fmt.Errorf("proxy port must be between 1 and 65535, got: %d", port) + } + + return &ProxyConfig{ + Enabled: true, + Type: proxyType, + Host: host, + Port: port, + Username: username, + Password: password, + }, nil +} + +const query_getArticlesBatchWithDateFilter_selectPart = `SELECT article_num, message_id, subject, from_header, date_sent, date_string, "references", bytes, lines, reply_count, path, headers_json, body_text, imported_at FROM articles` +const query_getArticlesBatchWithDateFilter_nodatefilter = `SELECT article_num, message_id, subject, from_header, date_sent, date_string, "references", bytes, lines, reply_count, path, headers_json, body_text, imported_at FROM articles ORDER BY date_sent ASC LIMIT ? OFFSET ?` +const query_getArticlesBatchWithDateFilter_orderby = " ORDER BY date_sent ASC LIMIT ? OFFSET ?" + +// getArticlesBatchWithDateFilter retrieves articles from a group database with optional date filtering +func getArticlesBatchWithDateFilter(groupDBs *database.GroupDBs, offset int64, startTime, endTime *time.Time) ([]*models.Article, error) { + + var query string + var args []interface{} + + if startTime != nil || endTime != nil { + // Build query with date filtering + + var whereConditions []string + + if startTime != nil { + whereConditions = append(whereConditions, "date_sent >= ?") + args = append(args, startTime.UTC().Format("2006-01-02 15:04:05")) + } + + if endTime != nil { + whereConditions = append(whereConditions, "date_sent <= ?") + args = append(args, endTime.UTC().Format("2006-01-02 15:04:05")) + } + + whereClause := "" + if len(whereConditions) > 0 { + whereClause = " WHERE " + strings.Join(whereConditions, " AND ") + } + + query = query_getArticlesBatchWithDateFilter_selectPart + whereClause + query_getArticlesBatchWithDateFilter_orderby + args = append(args, dbBatchSize, offset) + } else { + // No date filtering, use original query but with date_sent ordering + query = query_getArticlesBatchWithDateFilter_nodatefilter + args = []interface{}{dbBatchSize, offset} + } + + rows, err := groupDBs.DB.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []*models.Article + for rows.Next() { + var a models.Article + var artnum int64 + if err := rows.Scan(&artnum, &a.MessageID, &a.Subject, &a.FromHeader, &a.DateSent, &a.DateString, &a.References, &a.Bytes, &a.Lines, &a.ReplyCount, &a.Path, &a.HeadersJSON, &a.BodyText, &a.ImportedAt); err != nil { + return nil, err + } + a.ArticleNums = make(map[*string]int64) + out = append(out, &a) + } + + return out, nil +} + +// getArticleCountWithDateFilter gets the total count of articles with optional date filtering +func getArticleCountWithDateFilter(groupDBs *database.GroupDBs, startTime, endTime *time.Time) (int64, error) { + var query string + var args []interface{} + + if startTime != nil || endTime != nil { + // Build count query with date filtering + var whereConditions []string + + if startTime != nil { + whereConditions = append(whereConditions, "date_sent >= ?") + args = append(args, startTime.UTC().Format("2006-01-02 15:04:05")) + } + + if endTime != nil { + whereConditions = append(whereConditions, "date_sent <= ?") + args = append(args, endTime.UTC().Format("2006-01-02 15:04:05")) + } + + whereClause := "" + if len(whereConditions) > 0 { + whereClause = " WHERE " + strings.Join(whereConditions, " AND ") + } + + query = "SELECT COUNT(*) FROM articles" + whereClause + } else { + // No date filtering + query = "SELECT COUNT(*) FROM articles" + } + + var count int64 + err := groupDBs.DB.QueryRow(query, args...).Scan(&count) + if err != nil { + return 0, err + } + + return count, nil +} + +// testConnection tests the connection to the target NNTP server +func testConnection(host *string, port *int, username *string, password *string, ssl *bool, timeout *int, proxyConfig *ProxyConfig) error { + testProvider := &config.Provider{ + Name: "test", + Host: *host, + Port: *port, + SSL: *ssl, + Username: *username, + Password: *password, + MaxConns: 1, + Enabled: true, + Priority: 1, + } + + backendConfig := &nntp.BackendConfig{ + Host: *host, + Port: *port, + SSL: *ssl, + Username: *username, + Password: *password, + MaxConns: 1, + Provider: testProvider, + ConnectTimeout: time.Duration(*timeout) * time.Second, + } + + // Apply proxy configuration if specified + if proxyConfig != nil { + backendConfig.ProxyEnabled = proxyConfig.Enabled + backendConfig.ProxyType = proxyConfig.Type + backendConfig.ProxyHost = proxyConfig.Host + backendConfig.ProxyPort = proxyConfig.Port + backendConfig.ProxyUsername = proxyConfig.Username + backendConfig.ProxyPassword = proxyConfig.Password + } + + fmt.Printf("Testing connection to %s:%d (SSL: %v)\n", *host, *port, *ssl) + if *username != "" { + fmt.Printf("Authentication: %s\n", *username) + } else { + fmt.Println("Authentication: None") + } + if proxyConfig != nil { + fmt.Printf("Proxy: %s %s:%d\n", strings.ToUpper(proxyConfig.Type), proxyConfig.Host, proxyConfig.Port) + if proxyConfig.Username != "" { + fmt.Printf("Proxy Authentication: %s\n", proxyConfig.Username) + } + } + + // Test connection + client := nntp.NewConn(backendConfig) + start := time.Now() + err := client.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %v", err) + } + fmt.Printf("✓ Connection successful (took %v)\n", time.Since(start)) + client.CloseFromPoolOnly() // only use this in a test! + + return nil +} + +// getNewsgroupsToTransfer returns newsgroups matching the specified pattern +func getNewsgroupsToTransfer(db *database.Database, groupPattern string) ([]*models.Newsgroup, error) { + var newsgroups []*models.Newsgroup + + // Handle wildcard patterns + suffixWildcard := strings.HasSuffix(groupPattern, "*") + var wildcardPrefix string + + if suffixWildcard { + wildcardPrefix = strings.TrimSuffix(groupPattern, "*") + log.Printf("Using wildcard newsgroup prefix: '%s'", wildcardPrefix) + } + + // Get all newsgroups from database + allNewsgroups, err := db.MainDBGetAllNewsgroups() + if err != nil { + return nil, fmt.Errorf("failed to get newsgroups from database: %v", err) + } + + // Filter newsgroups based on pattern + if suffixWildcard { + for _, ng := range allNewsgroups { + if strings.HasPrefix(ng.Name, wildcardPrefix) { + newsgroups = append(newsgroups, ng) + } + } + } else { + // Exact match + for _, ng := range allNewsgroups { + if ng.Name == groupPattern { + newsgroups = append(newsgroups, ng) + break + } + } + } + + return newsgroups, nil +} + +// runTransfer performs the actual article transfer process +func runTransfer(db *database.Database, proc *processor.Processor, pool *nntp.Pool, newsgroups []*models.Newsgroup, batchCheck int, maxThreads int, dryRun bool, startTime, endTime *time.Time, shutdownChan <-chan struct{}) error { + + var totalTransferred int64 + var transferMutex sync.Mutex + maxThreadsChan := make(chan struct{}, maxThreads) + var wg sync.WaitGroup + // Process each newsgroup + log.Printf("Starting transfer for %d newsgroups", len(newsgroups)) + time.Sleep(3 * time.Second) + for _, newsgroup := range newsgroups { + if proc.WantShutdown(shutdownChan) { + log.Printf("Shutdown requested, stopping transfer. Total transferred: %d articles", totalTransferred) + return nil + } + maxThreadsChan <- struct{}{} // acquire a thread slot + wg.Add(1) + go func(ng *models.Newsgroup, wg *sync.WaitGroup) { + defer func(wg *sync.WaitGroup) { + wg.Done() + <-maxThreadsChan // release the thread slot + }(wg) + if proc.WantShutdown(shutdownChan) { + return + } + start := time.Now() + log.Printf("Starting transfer for newsgroup: %s", newsgroup.Name) + transferred, err := transferNewsgroup(db, proc, pool, newsgroup, batchCheck, dryRun, startTime, endTime, shutdownChan) + + transferMutex.Lock() + totalTransferred += transferred + transferMutex.Unlock() + + if err != nil { + log.Printf("Error transferring newsgroup %s: %v", newsgroup.Name, err) + } else { + log.Printf("Completed transfer for newsgroup %s: transferred %d articles. took %v", + newsgroup.Name, transferred, time.Since(start)) + } + }(newsgroup, &wg) + } + + // Wait for all transfers to complete + wg.Wait() + + log.Printf("Transfer summary: %d articles transferred", totalTransferred) + return nil +} + +type takeThisMode struct { + takeThisSuccessCount int + takeThisTotalCount int + useCheckMode bool // Start with TAKETHIS mode (false) +} + +// transferNewsgroup transfers articles from a single newsgroup +func transferNewsgroup(db *database.Database, proc *processor.Processor, pool *nntp.Pool, newsgroup *models.Newsgroup, batchCheck int, dryRun bool, startTime, endTime *time.Time, shutdownChan <-chan struct{}) (int64, error) { + + // Get group database + groupDBs, err := db.GetGroupDBs(newsgroup.Name) + if err != nil { + return 0, fmt.Errorf("failed to get group DBs for newsgroup '%s': %v", newsgroup.Name, err) + } + defer func() { + if ferr := db.ForceCloseGroupDBs(groupDBs); ferr != nil { + log.Printf("ForceCloseGroupDBs error for '%s': %v", newsgroup.Name, ferr) + } + }() + + // Get total article count first with date filtering + totalArticles, err := getArticleCountWithDateFilter(groupDBs, startTime, endTime) + if err != nil { + return 0, fmt.Errorf("failed to get article count for newsgroup '%s': %v", newsgroup.Name, err) + } + + if totalArticles == 0 { + if startTime != nil || endTime != nil { + log.Printf("No articles found in newsgroup: %s (within specified date range)", newsgroup.Name) + } else { + log.Printf("No articles found in newsgroup: %s", newsgroup.Name) + } + return 0, nil + } + + if dryRun { + if startTime != nil || endTime != nil { + log.Printf("DRY RUN: Would transfer %d articles from newsgroup %s (within specified date range)", totalArticles, newsgroup.Name) + } else { + log.Printf("DRY RUN: Would transfer %d articles from newsgroup %s", totalArticles, newsgroup.Name) + } + return 0, nil + } + + if startTime != nil || endTime != nil { + log.Printf("Found %d articles in newsgroup %s (within specified date range) - processing in batches", totalArticles, newsgroup.Name) + } else { + log.Printf("Found %d articles in newsgroup %s - processing in batches", totalArticles, newsgroup.Name) + } + //time.Sleep(3 * time.Second) // debug sleep + var transferred, ioffset int64 + remainingArticles := totalArticles + // Process articles in database batches (much larger than network batches) + ttMode := &takeThisMode{} + + for offset := ioffset; offset < totalArticles; offset += dbBatchSize { + if proc.WantShutdown(shutdownChan) { + log.Printf("WantShutdown in newsgroup: %s: Transferred %d articles", newsgroup.Name, transferred) + return transferred, nil + } + + // Load batch from database with date filtering + articles, err := getArticlesBatchWithDateFilter(groupDBs, offset, startTime, endTime) + if err != nil { + log.Printf("Error loading article batch (offset %d) for newsgroup %s: %v", offset, newsgroup.Name, err) + continue + } + + if len(articles) == 0 { + log.Printf("No more articles in newsgroup %s (offset %d)", newsgroup.Name, offset) + break + } + log.Printf("Newsgroup %s: Loaded %d articles from database (offset %d)", newsgroup.Name, len(articles), offset) + isleep := time.Second + // Process articles in network batches + for i := 0; i < len(articles); i += batchCheck { + if proc.WantShutdown(shutdownChan) { + log.Printf("WantShutdown in newsgroup: %s: Transferred %d articles", newsgroup.Name, transferred) + return transferred, nil + } + if !ttMode.useCheckMode && ttMode.takeThisTotalCount >= 100 { + ttMode.takeThisSuccessCount = 0 + ttMode.takeThisTotalCount = 0 + } + // Determine end index for the batch + end := i + batchCheck + if end > len(articles) { + end = len(articles) + } + forever: + for { + if proc.WantShutdown(shutdownChan) { + log.Printf("WantShutdown in newsgroup: %s: Transferred %d articles", newsgroup.Name, transferred) + return transferred, nil + } + // Get connection from pool + conn, err := pool.Get() + if err != nil { + return transferred, fmt.Errorf("failed to get connection from pool: %v", err) + } + batchTransferred, berr := processBatch(conn, newsgroup.Name, ttMode, articles[i:end]) + if berr != nil { + conn = nil + pool.Put(conn) + log.Printf("Error processing network batch for newsgroup %s: %v ... retry in %v", newsgroup.Name, err, isleep) + time.Sleep(isleep) + isleep = time.Duration(int64(isleep) * 2) + if isleep > time.Minute { + isleep = time.Minute + } + continue forever + } + isleep = time.Second + pool.Put(conn) + transferred += batchTransferred + log.Printf("Newsgroup %s: batch %d-%d processed (offset %d/%d) transferred %d", newsgroup.Name, i+1, end, offset, totalArticles, batchTransferred) + break forever + } + } + + // Clear articles slice to free memory + for i := range articles { + articles[i] = nil + } + remainingArticles -= int64(len(articles)) + articles = nil + + // todo verbose flag + log.Printf("Newsgroup %s: done (offset %d/%d), total transferred: %d, remainingArticles %d", newsgroup.Name, offset, totalArticles, transferred, remainingArticles) + } + + log.Printf("Completed newsgroup %s: total transferred: %d articles / total articles: %d", newsgroup.Name, transferred, totalArticles) + return transferred, nil +} + +var lowerLevel float64 = 90.0 +var upperLevel float64 = 95.0 + +// processBatch processes a batch of articles using NNTP streaming protocol (RFC 4644) +// Uses TAKETHIS primarily, falls back to CHECK when success rate < 95% +func processBatch(conn *nntp.BackendConn, newsgroupName string, ttMode *takeThisMode, articles []*models.Article) (int64, error) { + + if len(articles) == 0 { + return 0, nil + } + + // Calculate success rate to determine whether to use CHECK or TAKETHIS + var successRate float64 = 100.0 // Start optimistic + if ttMode.takeThisTotalCount > 0 { + successRate = float64(ttMode.takeThisSuccessCount) / float64(ttMode.takeThisTotalCount) * 100.0 + } + + // Switch to CHECK mode if TAKETHIS success rate drops below lowerLevel + if successRate < lowerLevel && ttMode.takeThisTotalCount >= 10 { // Need at least 10 attempts for meaningful stats + ttMode.useCheckMode = true + //log.Printf("newsgroup %s: TAKETHIS success rate %.1f%% < %d%%, switching to CHECK mode", newsgroupName, successRate, lowerLevel) + } else if successRate >= upperLevel && ttMode.takeThisTotalCount >= 20 { // Switch back when rate improves + ttMode.useCheckMode = false + //log.Printf("newsgroup %s: TAKETHIS success rate %.1f%% >= %d%%, switching back to TAKETHIS mode", newsgroupName, successRate, upperLevel) + } + + articleMap := make(map[string]*models.Article) + for _, article := range articles { + articleMap[article.MessageID] = article + } + + var transferred int64 + + if ttMode.useCheckMode { + // CHECK mode: verify articles are wanted before sending + log.Printf("Using CHECK mode for %d articles (success rate: %.1f%%)", len(articles), successRate) + + messageIds := make([]*string, len(articles)) + for i, article := range articles { + messageIds[i] = &article.MessageID + } + + // Send CHECK commands for all message IDs + checkResponses, err := conn.CheckMultiple(messageIds) + if err != nil { + return transferred, fmt.Errorf("failed to send CHECK command: %v", err) + } + + // Find wanted articles + wantedIds := make([]*string, 0) + for _, response := range checkResponses { + if response.Wanted { + wantedIds = append(wantedIds, response.MessageID) + } else { + log.Printf("Article %s not wanted by server: %d", *response.MessageID, response.Code) + } + } + + if len(wantedIds) == 0 { + log.Printf("No articles wanted by server in this batch") + if !ttMode.useCheckMode { + ttMode.useCheckMode = true + ttMode.takeThisSuccessCount = 0 + ttMode.takeThisTotalCount = len(messageIds) + } + return transferred, nil + } + if ttMode.useCheckMode && len(wantedIds) == len(messageIds) { + // use TAKETHIS mode if all articles are wanted + ttMode.useCheckMode = false + ttMode.takeThisSuccessCount = 0 + ttMode.takeThisTotalCount = 0 + } + log.Printf("Newsgroup: '%s' Server wants %d out of %d articles in batch", newsgroupName, len(wantedIds), len(messageIds)) + + // Send TAKETHIS for wanted articles + for _, msgId := range wantedIds { + count, err := sendArticleViaTakeThis(conn, articleMap[*msgId], ttMode) + if err != nil { + log.Printf("Failed to send TAKETHIS for %s: %v", *msgId, err) + continue + } + transferred += int64(count) + } + } else { + // TAKETHIS mode: send articles directly and track success rate + log.Printf("Using TAKETHIS mode for %d articles (success rate: %.1f%%)", len(articles), successRate) + + transferred, err := sendArticlesBatchViaTakeThis(conn, articles, ttMode) + if err != nil { + return int64(transferred), fmt.Errorf("failed to send TAKETHIS batch: %v", err) + } + if transferred == 0 { + if !ttMode.useCheckMode { + ttMode.useCheckMode = true + ttMode.takeThisSuccessCount = 0 + ttMode.takeThisTotalCount = len(articles) + } + } + return int64(transferred), nil + } + + return transferred, nil +} + +// sendArticlesBatchViaTakeThis sends multiple articles via TAKETHIS in streaming mode +// Sends all TAKETHIS commands first, then reads all responses (true streaming) +func sendArticlesBatchViaTakeThis(conn *nntp.BackendConn, articles []*models.Article, ttMode *takeThisMode) (int, error) { + if len(articles) == 0 { + return 0, nil + } + + // Phase 1: Send all TAKETHIS commands without waiting for responses + log.Printf("Phase 1: Sending %d TAKETHIS commands...", len(articles)) + + commandIDs := make([]uint, 0, len(articles)) + validArticles := make([]*models.Article, 0, len(articles)) + + for _, article := range articles { + // Send TAKETHIS command with article content (non-blocking) + cmdID, err := conn.SendTakeThisArticleStreaming(article, &processor.LocalNNTPHostname) + if err != nil { + log.Printf("Failed to send TAKETHIS for %s: %v", article.MessageID, err) + continue + } + + commandIDs = append(commandIDs, cmdID) + validArticles = append(validArticles, article) + } + + log.Printf("Sent %d TAKETHIS commands, reading responses...", len(commandIDs)) + + // Phase 2: Read all responses in order + transferred := 0 + for i, cmdID := range commandIDs { + article := validArticles[i] + + takeThisResponseCode, err := conn.ReadTakeThisResponseStreaming(cmdID) + if err != nil { + log.Printf("Failed to read TAKETHIS response for %s: %v", article.MessageID, err) + continue + } + + // Update success rate tracking + ttMode.takeThisTotalCount++ + if takeThisResponseCode == 239 { + ttMode.takeThisSuccessCount++ + transferred++ + } else { + log.Printf("Failed to transfer article %s: %d", article.MessageID, takeThisResponseCode) + } + } + + log.Printf("Batch transfer complete: %d/%d articles transferred successfully", transferred, len(articles)) + return transferred, nil +} + +// sendArticleViaTakeThis sends a single article via TAKETHIS and tracks success rate +func sendArticleViaTakeThis(conn *nntp.BackendConn, article *models.Article, ttMode *takeThisMode) (int, error) { + + // Send TAKETHIS command with article content + takeThisResponseCode, err := conn.TakeThisArticle(article, &processor.LocalNNTPHostname) + if err != nil { + return 0, fmt.Errorf("failed to send TAKETHIS: %v", err) + } + + // Update success rate tracking + ttMode.takeThisTotalCount++ + if takeThisResponseCode == 239 { + ttMode.takeThisSuccessCount++ + //log.Printf("Successfully transferred article: %s", article.MessageID) + return 1, nil + } else { + log.Printf("Failed to transfer article %s: %d", article.MessageID, takeThisResponseCode) + return 0, nil + } +} diff --git a/cmd/nntpmgr/main.go b/cmd/nntpmgr/main.go index f94f826b..229302aa 100644 --- a/cmd/nntpmgr/main.go +++ b/cmd/nntpmgr/main.go @@ -19,6 +19,7 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("go-pugleaf NNTP User Manager (version: %s)", config.AppVersion) var ( createUser = flag.Bool("create", false, "Create a new NNTP user") @@ -29,11 +30,10 @@ func main() { password = flag.String("password", "random", "Password for NNTP user (10-20 chars, will be bcrypt hashed)") maxConns = flag.Int("maxconns", 1, "Maximum concurrent connections") posting = flag.Bool("posting", false, "Allow posting (default: read-only)") - newsgroup = flag.String("rescan-db", "", "[TOOL] Rescan database for newsgroup (default: alt.test)") ) flag.Parse() - if !*createUser && !*listUsers && !*deleteUser && !*updateUser && *newsgroup == "" { + if !*createUser && !*listUsers && !*deleteUser && !*updateUser { fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nOptions:\n") flag.PrintDefaults() @@ -43,7 +43,6 @@ func main() { fmt.Fprintf(os.Stderr, " %s -list\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s -update -username reader1 -maxconns 5 -posting\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s -delete -username reader1\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s -rescan-db alt.test\n", os.Args[0]) os.Exit(1) } @@ -80,16 +79,6 @@ func main() { log.Fatalf("Failed to apply database migrations: %v", err) } - if *newsgroup != "" { - // Rescan database for the specified newsgroup - log.Printf("Rescanning database for newsgroup '%s'...", *newsgroup) - if err := db.Rescan(*newsgroup); err != nil { - log.Fatalf("Failed to rescan newsgroup '%s': %v", *newsgroup, err) - } - log.Printf("✅ Rescan completed for newsgroup '%s'", *newsgroup) - return - } - switch { case *createUser: if *username == "" { diff --git a/cmd/parsedates/main.go b/cmd/parsedates/main.go index 434b3b05..bc6645e5 100644 --- a/cmd/parsedates/main.go +++ b/cmd/parsedates/main.go @@ -7,6 +7,7 @@ import ( "time" "github.com/go-while/go-pugleaf/internal/config" + "github.com/go-while/go-pugleaf/internal/database" "github.com/go-while/go-pugleaf/internal/processor" ) @@ -14,6 +15,7 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("go-pugleaf Date Parser (version: %s)", config.AppVersion) if len(os.Args) < 2 { fmt.Println("Usage: parsedates \"date string\"") diff --git a/cmd/post-queue/main.go b/cmd/post-queue/main.go new file mode 100644 index 00000000..05f31257 --- /dev/null +++ b/cmd/post-queue/main.go @@ -0,0 +1,198 @@ +// Post-queue tool for go-pugleaf +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "sync" + "time" + + "github.com/go-while/go-pugleaf/internal/config" + "github.com/go-while/go-pugleaf/internal/database" + "github.com/go-while/go-pugleaf/internal/models" + "github.com/go-while/go-pugleaf/internal/nntp" + "github.com/go-while/go-pugleaf/internal/postmgr" +) + +var appVersion = "-unset-" + +func main() { + config.AppVersion = appVersion + database.DBidleTimeOut = 15 * time.Second + database.NO_CACHE_BOOT = true // prevents booting caches + log.Printf("Starting go-pugleaf Post Queue Tool (version %s)", config.AppVersion) + + // Command line flags + var ( + showHelp = flag.Bool("help", false, "Show usage examples and exit") + daemon = flag.Bool("daemon", false, "Run as a daemon") + limit = flag.Int("max-batch", 100, "Post max N articles in a batch") + ) + flag.Parse() + + if *showHelp { + showUsageExamples() + os.Exit(0) + } + + // Initialize database + db, err := database.OpenDatabase(nil) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer db.Shutdown() + + // Set up signal handling for graceful shutdown + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt) + shutdownChan := make(chan struct{}) + + go func() { + <-sigChan + log.Printf("Received shutdown signal, initiating graceful shutdown...") + close(shutdownChan) + }() + + // Get providers with posting enabled + providers, err := getPostingProviders(db) + if err != nil { + log.Fatalf("Failed to get posting providers: %v", err) + } + + if len(providers) == 0 { + log.Printf("No providers with posting enabled found") + os.Exit(0) + } + + log.Printf("Found %d providers with posting enabled", len(providers)) + + // Create connection pools for posting providers + pools, err := createPostingPools(providers) + if err != nil { + log.Fatalf("Failed to create posting pools: %v", err) + } + defer func() { + for _, pool := range pools { + pool.ClosePool() + } + }() + + // Create poster manager + posterManager := postmgr.NewPosterManager(db, pools) + + // Start the posting loop + log.Printf("Starting post queue processing...") + + if !*daemon { + done, err := posterManager.ProcessPendingPosts(*limit) + if err != nil { + log.Printf("Error processing pending posts: %v", err) + os.Exit(1) + } + log.Printf("Processed %d pending posts", done) + os.Exit(0) + } + var wg sync.WaitGroup + wg.Add(1) + go posterManager.Run(*limit, shutdownChan, &wg) + wg.Wait() + + log.Printf("Post Queue Tool has shut down gracefully") + +} + +// showUsageExamples displays usage examples +func showUsageExamples() { + fmt.Println("\n=== Post Queue Tool - Usage Examples ===") + fmt.Println("The Post Queue tool processes articles queued from the web interface") + fmt.Println("and posts them to all providers with posting enabled.") + fmt.Println() + fmt.Println("Basic Usage:") + fmt.Println(" ./post-queue") + fmt.Println() + fmt.Println("Advanced Options:") + fmt.Println(" ./post-queue -max-providers 5 -max-concurrent 3") + fmt.Println(" ./post-queue -retry-attempts 5 -retry-delay 60s") + fmt.Println(" ./post-queue -check-interval 30s") + fmt.Println(" ./post-queue -dry-run (test mode without actual posting)") + fmt.Println() + fmt.Println("Monitoring:") + fmt.Println(" ./post-queue -max-concurrent 1 -check-interval 10s") + fmt.Println() +} + +// getPostingProviders returns all enabled providers with posting capability +func getPostingProviders(db *database.Database) ([]*models.Provider, error) { + allProviders, err := db.GetProviders() + if err != nil { + return nil, err + } + + var postingProviders []*models.Provider + for _, p := range allProviders { + if p.Enabled && p.Posting && p.Host != "" && p.Port > 0 && p.MaxConns > 0 { + log.Printf("Found posting provider: %s (ID: %d, Host: %s, Port: %d, MaxConns: %d)", + p.Name, p.ID, p.Host, p.Port, p.MaxConns) + postingProviders = append(postingProviders, p) + } + } + + return postingProviders, nil +} + +// createPostingPools creates NNTP connection pools for posting providers +func createPostingPools(providers []*models.Provider) ([]*nntp.Pool, error) { + pools := make([]*nntp.Pool, 0, len(providers)) + + for _, p := range providers { + // Convert models.Provider to config.Provider + configProvider := &config.Provider{ + Grp: p.Grp, + Name: p.Name, + Host: p.Host, + Port: p.Port, + SSL: p.SSL, + Username: p.Username, + Password: p.Password, + MaxConns: p.MaxConns, + Enabled: p.Enabled, + Priority: p.Priority, + MaxArtSize: p.MaxArtSize, + Posting: p.Posting, + // Copy proxy configuration + ProxyEnabled: p.ProxyEnabled, + ProxyType: p.ProxyType, + ProxyHost: p.ProxyHost, + ProxyPort: p.ProxyPort, + ProxyUsername: p.ProxyUsername, + ProxyPassword: p.ProxyPassword, + } + + backendConfig := &nntp.BackendConfig{ + Host: p.Host, + Port: p.Port, + SSL: p.SSL, + Username: p.Username, + Password: p.Password, + MaxConns: p.MaxConns, + Provider: configProvider, + // Copy proxy configuration + ProxyEnabled: p.ProxyEnabled, + ProxyType: p.ProxyType, + ProxyHost: p.ProxyHost, + ProxyPort: p.ProxyPort, + ProxyUsername: p.ProxyUsername, + ProxyPassword: p.ProxyPassword, + } + + pool := nntp.NewPool(backendConfig) + pool.StartCleanupWorker(5 * time.Second) + pools = append(pools, pool) + log.Printf("Created posting pool for provider '%s' with max %d connections", p.Name, p.MaxConns) + } + + return pools, nil +} diff --git a/cmd/recover-db/main.go b/cmd/recover-db/main.go index 51ae799e..a7e62ca5 100644 --- a/cmd/recover-db/main.go +++ b/cmd/recover-db/main.go @@ -6,6 +6,8 @@ import ( "fmt" "log" "os" + "strings" + "sync" "time" "github.com/go-while/go-pugleaf/internal/config" @@ -28,14 +30,17 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("go-pugleaf Database Recovery Tool (version: %s)", config.AppVersion) var ( - dbPath = flag.String("db", "data", "Data Path to main data directory (required)") - newsgroup = flag.String("group", "$all", "Newsgroup name to check (required) (\\$all to check for all)") - verbose = flag.Bool("v", true, "Verbose output") - repair = flag.Bool("repair", false, "Attempt to repair detected inconsistencies") - parseDates = flag.Bool("parsedates", false, "Check and log date parsing differences between date_string and date_sent") - rewriteDates = flag.Bool("rewritedates", false, "Rewrite incorrect dates (requires -parsedates)") + dbPath = flag.String("db", "data", "Data Path to main data directory (required)") + newsgroup = flag.String("group", "$all", "Newsgroup name to check (required) (\\$all to check for all or news.* to check for all in that hierarchy)") + verbose = flag.Bool("v", true, "Verbose output") + repair = flag.Bool("repair", false, "Attempt to repair detected inconsistencies") + parseDates = flag.Bool("parsedates", false, "Check and log date parsing differences between date_string and date_sent") + rewriteDates = flag.Bool("rewritedates", false, "Rewrite incorrect dates (requires -parsedates)") + rebuildThreads = flag.Bool("rebuild-threads", false, "Rebuild all thread relationships from scratch (destructive)") + maxPar = flag.Int("max-par", 1, "use with -rebuild-threads to process N newsgroups") ) flag.Parse() @@ -79,11 +84,36 @@ func main() { log.Fatalf("Failed to apply database migrations: %v", err) } var newsgroups []*models.Newsgroup - if *newsgroup != "$all" && *newsgroup != "" { - newsgroups = append(newsgroups, &models.Newsgroup{ - Name: *newsgroup, - }) + isWildcard := strings.HasSuffix(*newsgroup, "*") + if *newsgroup != "$all" && *newsgroup != "" && !isWildcard { + // if is comma separated check for multiple newsgroups + if strings.Contains(*newsgroup, ",") { + for _, grpName := range strings.Split(*newsgroup, ",") { + if grpName == "" { + continue + } + newsgroups = append(newsgroups, &models.Newsgroup{ + Name: strings.TrimSpace(grpName), + }) + } + } else { + newsgroups = append(newsgroups, &models.Newsgroup{ + Name: *newsgroup, + }) + } + } else if isWildcard { + // strip * from newsgroup and add only newsgroups matching the strings prefix + prefix := strings.TrimSuffix(*newsgroup, "*") + allGroups, err := db.MainDBGetAllNewsgroups() + if err != nil { + log.Fatalf("failed to get newsgroups from database: %v", err) + } + for _, grp := range allGroups { + if strings.HasPrefix(grp.Name, prefix) { + newsgroups = append(newsgroups, grp) + } + } } else { newsgroups, err = db.MainDBGetAllNewsgroups() if err != nil { @@ -95,9 +125,48 @@ func main() { fmt.Printf("📂 Data Path: %s\n", *dbPath) fmt.Printf("📊 Newsgroups: %d\n", len(newsgroups)) fmt.Printf("🔧 Repair Mode: %v\n", *repair) + fmt.Printf("🧵 Rebuild Threads: %v\n", *rebuildThreads) fmt.Printf("📅 Parse Dates: %v\n", *parseDates) fmt.Printf("🔄 Rewrite Dates: %v\n", *rewriteDates) fmt.Printf("\n") + parChan := make(chan struct{}, *maxPar) + var parMux sync.Mutex + var wg sync.WaitGroup + // If only thread rebuilding is requested, run that and exit + if *rebuildThreads { + start := time.Now() + fmt.Printf("🧵 Starting thread rebuild process...\n") + fmt.Printf("=====================================\n") + var totalArticles, totalThreadsRebuilt int64 + for i, newsgroup := range newsgroups { + parChan <- struct{}{} // get lock + wg.Add(1) + go func(newsgroup *models.Newsgroup, wg *sync.WaitGroup) { + defer func(wg *sync.WaitGroup) { + <-parChan // release lock + wg.Done() + }(wg) + fmt.Printf("🧵 [%d/%d] Rebuilding threads for newsgroup: %s\n", i+1, len(newsgroups), newsgroup.Name) + report, err := db.RebuildThreadsFromScratch(newsgroup.Name, *verbose) + if err != nil { + fmt.Printf("❌ Failed to rebuild threads for '%s': %v\n", newsgroup.Name, err) + return + } + report.PrintReport() + parMux.Lock() + totalArticles += report.TotalArticles + totalThreadsRebuilt += report.ThreadsRebuilt + parMux.Unlock() + }(newsgroup, &wg) + } + wg.Wait() + parMux.Lock() + fmt.Printf("\n🧵 Thread rebuild completed (%d newsgroups) took: %d ms\n", len(newsgroups), time.Since(start).Milliseconds()) + fmt.Printf(" Total articles processed: %d\n", totalArticles) + fmt.Printf(" Total threads rebuilt: %d\n", totalThreadsRebuilt) + parMux.Unlock() + os.Exit(0) + } // If only date parsing is requested, run that and exit if *parseDates { @@ -187,6 +256,18 @@ func main() { } fmt.Printf("🔍 Repair completed. Re-running consistency check...\n\n") + // Optionally rebuild threads after repair if there were thread-related issues + if len(report.OrphanedThreads) > 0 { + fmt.Printf("🧵 Rebuilding thread relationships after repair...\n") + threadReport, err := db.RebuildThreadsFromScratch(newsgroup.Name, *verbose) + if err != nil { + fmt.Printf("❌ Failed to rebuild threads: %v\n", err) + } else { + fmt.Printf("✅ Thread rebuild completed: %d threads rebuilt from %d articles\n", + threadReport.ThreadsRebuilt, threadReport.TotalArticles) + } + } + // Re-run consistency check after repair report, err = db.CheckDatabaseConsistency(newsgroup.Name) if err != nil { @@ -604,8 +685,8 @@ type DateProblem struct { } // checkAndFixDates analyzes date_string vs date_sent mismatches and optionally fixes them -func checkAndFixDates(db *database.Database, newsgroups []*models.Newsgroup, rewriteDates, verbose bool) (int, int, error) { - var totalFixed, totalChecked int +func checkAndFixDates(db *database.Database, newsgroups []*models.Newsgroup, rewriteDates, verbose bool) (int64, int64, error) { + var totalFixed, totalChecked int64 var allProblems []DateProblem for i, newsgroup := range newsgroups { @@ -654,7 +735,7 @@ func checkAndFixDates(db *database.Database, newsgroups []*models.Newsgroup, rew } // checkGroupDates checks and optionally fixes date mismatches in a single newsgroup -func checkGroupDates(groupDB *database.GroupDBs, newsgroupName string, rewriteDates, verbose bool) (int, int, []DateProblem, error) { +func checkGroupDates(groupDB *database.GroupDBs, newsgroupName string, rewriteDates, verbose bool) (int64, int64, []DateProblem, error) { // Query all articles with their date information - get date_sent as string to avoid timezone parsing issues rows, err := database.RetryableQuery(groupDB.DB, ` SELECT article_num, message_id, date_string, date_sent @@ -667,7 +748,7 @@ func checkGroupDates(groupDB *database.GroupDBs, newsgroupName string, rewriteDa } defer rows.Close() - var fixed, checked int + var fixed, checked int64 var problems []DateProblem var tx *sql.Tx @@ -694,7 +775,7 @@ func checkGroupDates(groupDB *database.GroupDBs, newsgroupName string, rewriteDa checked++ // print progress every N - if checked%25000 == 0 { + if checked%database.RescanBatchSize == 0 { fmt.Printf(" 📊 Checked %d articles so far...\n", checked) } diff --git a/cmd/rslight-importer/main.go b/cmd/rslight-importer/main.go index bd27f8d5..0d85af24 100644 --- a/cmd/rslight-importer/main.go +++ b/cmd/rslight-importer/main.go @@ -31,7 +31,7 @@ func main() { sqliteDir = flag.String("spool", "", "Path to legacy RockSolid spool directory containing SQLite files *.db3") threads = flag.Int("threads", 1, "parallel import threads (default: 1)") useShortHashLen = flag.Int("useshorthashlen", 7, "short hash length for history storage (2-7, default: 7) - NOTE: cannot be changed once set!") - hostnamePath = flag.String("nntphostname", "", "your hostname must be set") + nntphostname = flag.String("nntphostname", "", "your hostname must be set") ) flag.Parse() @@ -52,11 +52,7 @@ func main() { } mainConfig := config.NewDefaultConfig() mainConfig.AppVersion = appVersion - if *hostnamePath == "" { - log.Fatalf("[RSLIGHT-IMPORT]: Error: hostname must be set!") - } - mainConfig.Server.Hostname = *hostnamePath - processor.LocalHostnamePath = *hostnamePath + mainConfig.Server.Hostname = *nntphostname log.Printf("Starting go-pugleaf RSLIGHT-IMPORT (version: %s)", appVersion) // Create database configuration @@ -74,6 +70,11 @@ func main() { log.Fatalf("Failed to run database migrations: %v", err) } + // Set hostname in processor with database fallback support + if err := processor.SetHostname(*nntphostname, db); err != nil { + log.Fatalf("Failed to set NNTP hostname: %v", err) + } + // Handle UseShortHashLen configuration with locking lockedHashLen, isLocked, err := db.GetHistoryUseShortHashLen(*useShortHashLen) if err != nil { diff --git a/cmd/tcp2tor/build.sh b/cmd/tcp2tor/build.sh new file mode 100755 index 00000000..66498ce7 --- /dev/null +++ b/cmd/tcp2tor/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Build script for tcp2tor + +set -e + +echo "Building tcp2tor..." + +# Get version info +if [ -f "../../VERSION" ]; then + VERSION=$(cat ../../VERSION) +else + VERSION="dev-$(date +%Y%m%d)" +fi + +# Build for current platform +go build -trimpath -ldflags "-w -s -X main.appVersion=$VERSION" -o tcp2tor . || exit 1 +sha256sum tcp2tor > tcp2tor.sha256 +echo "✓ Built tcp2tor (version $VERSION)" +echo "" +echo "Usage examples:" +echo " ./tcp2tor -help" +echo "" diff --git a/cmd/tcp2tor/main.go b/cmd/tcp2tor/main.go new file mode 100644 index 00000000..1998a480 --- /dev/null +++ b/cmd/tcp2tor/main.go @@ -0,0 +1,400 @@ +// tcp2tor - General TCP proxy tool for go-pugleaf +// This tool creates a local TCP listener that forwards raw TCP connections through a SOCKS5 proxy +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/net/proxy" +) + +var appVersion = "-unset-" + +// ProxyConfig holds the configuration for the SOCKS5 proxy +type ProxyConfig struct { + SocksHost string + SocksPort int + SocksAuth *proxy.Auth // Optional authentication +} + +// showUsageExamples displays usage examples for tcp2tor +func showUsageExamples() { + fmt.Println("\n=== tcp2tor - General TCP Proxy Tool ===") + fmt.Println("Creates a local TCP listener that forwards raw TCP connections through SOCKS5 proxy.") + fmt.Println("Note: Works with any TCP service and any SOCKS5 proxy - not limited to NNTP or Tor.") + fmt.Println() + fmt.Println("Basic Usage:") + fmt.Println(" ./tcp2tor -listen-port 1119 -listen-host 127.2.3.4 -target test.onion:119") + fmt.Println(" ./tcp2tor -listen-port 1563 -listen-host 127.2.3.4 -target test.onion:563") + fmt.Println() + fmt.Println("Custom SOCKS5 Proxy:") + fmt.Println(" ./tcp2tor -listen-port 1119 -listen-host 127.2.3.4 -target test.onion:119 -socks5-host 127.0.0.1 -socks5-port 9050") + fmt.Println(" ./tcp2tor -listen-port 1119 -listen-host 127.2.3.4 -target test.onion:119 -socks5-proxy 192.168.1.100:9050") + fmt.Println() + fmt.Println("SOCKS5 Authentication:") + fmt.Println(" ./tcp2tor -listen-port 1119 -listen-host 127.2.3.4 -target test.onion:119 -socks5-user myuser -socks5-pass mypass") + fmt.Println() + fmt.Println("Multiple Targets (use multiple instances):") + fmt.Println(" ./tcp2tor -listen-port 1119 -listen-host 127.2.3.4 -target news1.onion:119 &") + fmt.Println(" ./tcp2tor -listen-port 1120 -listen-host 127.2.3.5 -target news2.onion:119 &") + fmt.Println() + fmt.Println("Then configure your NNTP client to connect to localhost:1119") + fmt.Println() +} + +func main() { + log.Printf("Starting tcp2tor (version %s)", appVersion) + + // Command line flags + var ( + listenPort = flag.Int("listen-port", 1119, "Local port to listen on for incoming connections") + listenHost = flag.String("listen-host", "", "Local host/IP to bind to like 127.2.3.4") + targetAddr = flag.String("target", "", "Target onion address and port (e.g., example.onion:119)") + + // SOCKS5 proxy configuration + socksHost = flag.String("socks5-host", "127.0.0.1", "SOCKS5 proxy host") + socksPort = flag.Int("socks5-port", 9050, "SOCKS5 proxy port") + socksProxy = flag.String("socks5-proxy", "", "SOCKS5 proxy address (host:port) - overrides -socks5-host/-socks5-port") + socksUser = flag.String("socks5-user", "", "SOCKS5 proxy username (optional)") + socksPass = flag.String("socks5-pass", "", "SOCKS5 proxy password (optional)") + + // Operation options + showHelp = flag.Bool("help", false, "Show usage examples and exit") + timeout = flag.Int("timeout", 30, "Connection timeout in seconds") + verbose = flag.Bool("verbose", false, "Enable verbose logging") + ) + flag.Parse() + + // Show help if requested + if *showHelp { + showUsageExamples() + os.Exit(0) + } + + // Validate required flags + if *targetAddr == "" { + log.Fatalf("Error: -target must be specified (e.g., example.onion:119)") + } + + // Parse target address + targetHost, targetPort, err := parseTargetAddress(*targetAddr) + if err != nil { + log.Fatalf("Error: Invalid target address '%s': %v", *targetAddr, err) + } + + // Validate listen port + if *listenPort < 1 || *listenPort > 65535 { + log.Fatalf("Error: listen-port must be between 1 and 65535 (got %d)", *listenPort) + } + + // Parse SOCKS5 proxy configuration + proxyConfig, err := parseProxyConfig(*socksProxy, *socksHost, *socksPort, *socksUser, *socksPass) + if err != nil { + log.Fatalf("Error: Invalid SOCKS5 proxy configuration: %v", err) + } + + // Create listen address + listenAddr := fmt.Sprintf("%s:%d", *listenHost, *listenPort) + + log.Printf("Configuration:") + log.Printf(" Listen: %s", listenAddr) + log.Printf(" Target: %s:%s", targetHost, targetPort) + log.Printf(" SOCKS5 Proxy: %s:%d", proxyConfig.SocksHost, proxyConfig.SocksPort) + if proxyConfig.SocksAuth != nil { + log.Printf(" SOCKS5 Auth: %s", proxyConfig.SocksAuth.User) + } + log.Printf(" Timeout: %d seconds", *timeout) + + // Test SOCKS5 proxy connection + if err := testSOCKS5Connection(proxyConfig, targetHost, targetPort, *timeout, *verbose); err != nil { + log.Fatalf("Error: Failed to connect through SOCKS5 proxy: %v", err) + } + log.Printf("✓ SOCKS5 proxy connection test successful") + + // Start the proxy server + server := &ProxyServer{ + ListenAddr: listenAddr, + TargetHost: targetHost, + TargetPort: targetPort, + ProxyConfig: proxyConfig, + Timeout: time.Duration(*timeout) * time.Second, + Verbose: *verbose, + } + + // Set up signal handling for graceful shutdown + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + // Start server in goroutine + serverDone := make(chan error, 1) + go func() { + serverDone <- server.Start() + }() + + // Wait for shutdown signal or server error + select { + case sig := <-sigChan: + log.Printf("Received signal %v, shutting down gracefully...", sig) + server.Stop() + case err := <-serverDone: + if err != nil { + log.Fatalf("Server error: %v", err) + } + } + + log.Printf("tcp2tor proxy shutdown complete") +} + +// parseTargetAddress parses target address in format "host:port" +func parseTargetAddress(target string) (host, port string, err error) { + parts := strings.Split(target, ":") + if len(parts) != 2 { + return "", "", fmt.Errorf("target must be in format 'host:port'") + } + + host = strings.TrimSpace(parts[0]) + port = strings.TrimSpace(parts[1]) + + if host == "" { + return "", "", fmt.Errorf("host cannot be empty") + } + + // Validate port + if portNum, err := strconv.Atoi(port); err != nil || portNum < 1 || portNum > 65535 { + return "", "", fmt.Errorf("port must be a number between 1 and 65535") + } + + return host, port, nil +} + +// parseProxyConfig creates proxy configuration from command line flags +func parseProxyConfig(socksProxy, socksHost string, socksPort int, socksUser, socksPass string) (*ProxyConfig, error) { + config := &ProxyConfig{} + + // Parse proxy address + if socksProxy != "" { + // Use -socks5-proxy flag (overrides host/port) + parts := strings.Split(socksProxy, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("socks5-proxy must be in format 'host:port'") + } + config.SocksHost = strings.TrimSpace(parts[0]) + if port, err := strconv.Atoi(strings.TrimSpace(parts[1])); err != nil { + return nil, fmt.Errorf("invalid port in socks5-proxy: %v", err) + } else { + config.SocksPort = port + } + } else { + // Use individual host/port flags + config.SocksHost = socksHost + config.SocksPort = socksPort + } + + // Validate proxy configuration + if config.SocksHost == "" { + return nil, fmt.Errorf("SOCKS5 proxy host cannot be empty") + } + if config.SocksPort < 1 || config.SocksPort > 65535 { + return nil, fmt.Errorf("SOCKS5 proxy port must be between 1 and 65535") + } + + // Set up authentication if provided + if socksUser != "" || socksPass != "" { + config.SocksAuth = &proxy.Auth{ + User: socksUser, + Password: socksPass, + } + } + + return config, nil +} + +// testSOCKS5Connection tests the SOCKS5 proxy connection +func testSOCKS5Connection(config *ProxyConfig, targetHost, targetPort string, timeoutSec int, verbose bool) error { + // Create SOCKS5 dialer + proxyAddr := fmt.Sprintf("%s:%d", config.SocksHost, config.SocksPort) + + var dialer proxy.Dialer + var err error + + if config.SocksAuth != nil { + if verbose { + log.Printf("Creating SOCKS5 dialer with authentication to %s", proxyAddr) + } + dialer, err = proxy.SOCKS5("tcp", proxyAddr, config.SocksAuth, proxy.Direct) + } else { + if verbose { + log.Printf("Creating SOCKS5 dialer without authentication to %s", proxyAddr) + } + dialer, err = proxy.SOCKS5("tcp", proxyAddr, nil, proxy.Direct) + } + + if err != nil { + return fmt.Errorf("failed to create SOCKS5 dialer: %v", err) + } + + // Test connection + targetAddr := fmt.Sprintf("%s:%s", targetHost, targetPort) + if verbose { + log.Printf("Testing connection to %s through SOCKS5 proxy...", targetAddr) + } + + conn, err := dialer.Dial("tcp", targetAddr) + if err != nil { + return fmt.Errorf("failed to connect to %s through SOCKS5 proxy: %v", targetAddr, err) + } + defer conn.Close() + + if verbose { + log.Printf("Successfully connected to %s", targetAddr) + } + + return nil +} + +// ProxyServer handles the TCP proxy functionality +type ProxyServer struct { + ListenAddr string + TargetHost string + TargetPort string + ProxyConfig *ProxyConfig + Timeout time.Duration + Verbose bool + listener net.Listener + shutdown chan struct{} +} + +// Start starts the proxy server +func (s *ProxyServer) Start() error { + var err error + s.listener, err = net.Listen("tcp", s.ListenAddr) + if err != nil { + return fmt.Errorf("failed to listen on %s: %v", s.ListenAddr, err) + } + defer s.listener.Close() + + s.shutdown = make(chan struct{}) + log.Printf("tcp2tor proxy listening on %s", s.ListenAddr) + log.Printf("Forwarding connections to %s:%s through SOCKS5 proxy %s:%d", + s.TargetHost, s.TargetPort, s.ProxyConfig.SocksHost, s.ProxyConfig.SocksPort) + + for { + select { + case <-s.shutdown: + log.Printf("Proxy server shutting down...") + return nil + default: + } + + // Set accept timeout + if tcpListener, ok := s.listener.(*net.TCPListener); ok { + tcpListener.SetDeadline(time.Now().Add(1 * time.Second)) + } + + conn, err := s.listener.Accept() + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + continue // Timeout, check for shutdown + } + if strings.Contains(err.Error(), "use of closed network connection") { + return nil // Normal shutdown + } + return fmt.Errorf("failed to accept connection: %v", err) + } + + // Handle connection in goroutine + go s.handleConnection(conn) + } +} + +// Stop stops the proxy server +func (s *ProxyServer) Stop() { + if s.shutdown != nil { + close(s.shutdown) + } + if s.listener != nil { + s.listener.Close() + } +} + +// handleConnection handles a single client connection +func (s *ProxyServer) handleConnection(clientConn net.Conn) { + defer clientConn.Close() + + clientAddr := clientConn.RemoteAddr().String() + if s.Verbose { + log.Printf("New connection from %s", clientAddr) + } + + // Create SOCKS5 dialer + proxyAddr := fmt.Sprintf("%s:%d", s.ProxyConfig.SocksHost, s.ProxyConfig.SocksPort) + + var dialer proxy.Dialer + var err error + + if s.ProxyConfig.SocksAuth != nil { + dialer, err = proxy.SOCKS5("tcp", proxyAddr, s.ProxyConfig.SocksAuth, proxy.Direct) + } else { + dialer, err = proxy.SOCKS5("tcp", proxyAddr, nil, proxy.Direct) + } + + if err != nil { + log.Printf("Failed to create SOCKS5 dialer for %s: %v", clientAddr, err) + return + } + + // Connect to target through SOCKS5 proxy + targetAddr := fmt.Sprintf("%s:%s", s.TargetHost, s.TargetPort) + if s.Verbose { + log.Printf("Connecting to %s through SOCKS5 proxy for client %s", targetAddr, clientAddr) + } + + targetConn, err := dialer.Dial("tcp", targetAddr) + if err != nil { + log.Printf("Failed to connect to %s for client %s: %v", targetAddr, clientAddr, err) + return + } + defer targetConn.Close() + + if s.Verbose { + log.Printf("Connected to %s for client %s", targetAddr, clientAddr) + } + + // Start bidirectional forwarding + done := make(chan struct{}, 2) + + // Forward client -> target + go func() { + defer func() { done <- struct{}{} }() + written, err := io.Copy(targetConn, clientConn) + if s.Verbose && err != nil && !strings.Contains(err.Error(), "use of closed network connection") { + log.Printf("Client->Target copy error for %s: %v (wrote %d bytes)", clientAddr, err, written) + } + }() + + // Forward target -> client + go func() { + defer func() { done <- struct{}{} }() + written, err := io.Copy(clientConn, targetConn) + if s.Verbose && err != nil && !strings.Contains(err.Error(), "use of closed network connection") { + log.Printf("Target->Client copy error for %s: %v (wrote %d bytes)", clientAddr, err, written) + } + }() + + // Wait for either direction to close + <-done + + if s.Verbose { + log.Printf("Connection closed for client %s", clientAddr) + } +} diff --git a/cmd/usermgr/main.go b/cmd/usermgr/main.go index 610c8ca4..763d3d82 100644 --- a/cmd/usermgr/main.go +++ b/cmd/usermgr/main.go @@ -22,6 +22,7 @@ var appVersion = "-unset-" func main() { config.AppVersion = appVersion + database.NO_CACHE_BOOT = true // prevents booting caches log.Printf("go-pugleaf Web User Manager (version: %s)", config.AppVersion) var ( createUser = flag.Bool("create", false, "Create a new user") diff --git a/cmd/web/main.go b/cmd/web/main.go index 41154771..87071786 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -3,20 +3,16 @@ package main import ( "context" - "database/sql" "flag" - "fmt" "log" "net/http" "os" "os/signal" "path/filepath" - "sync" "time" "github.com/go-while/go-pugleaf/internal/config" "github.com/go-while/go-pugleaf/internal/database" - "github.com/go-while/go-pugleaf/internal/history" "github.com/go-while/go-pugleaf/internal/models" "github.com/go-while/go-pugleaf/internal/nntp" "github.com/go-while/go-pugleaf/internal/preloader" @@ -25,10 +21,9 @@ import ( ) var ( - webmutex sync.Mutex // command-line flags - hostnamePath string + nntphostname string isleep int64 webport int webssl bool @@ -62,6 +57,10 @@ var ( writeActiveFile string writeActiveOnly bool + // Compare flags + compareActiveFile string + compareActiveMinArticles int64 + // Bridge flags (disabled by default) /* code path disabled (not tested) enableFediverse bool @@ -74,323 +73,6 @@ var ( */ ) -// ProcessorAdapter adapts the processor.Processor to implement nntp.ArticleProcessor interface -type ProcessorAdapter struct { - processor *processor.Processor -} - -// NewProcessorAdapter creates a new processor adapter -func NewProcessorAdapter(proc *processor.Processor) *ProcessorAdapter { - return &ProcessorAdapter{processor: proc} -} - -// ProcessIncomingArticle processes an incoming article -func (pa *ProcessorAdapter) ProcessIncomingArticle(article *models.Article) (int, error) { - // Forward the Article directly to the processor - // No conversions needed since both use models.Article - return pa.processor.ProcessIncomingArticle(article) -} - -// Lookup checks if a message-ID exists in history -func (pa *ProcessorAdapter) Lookup(msgIdItem *history.MessageIdItem) (int, error) { - return pa.processor.History.Lookup(msgIdItem) -} - -// CheckNoMoreWorkInHistory checks if there's no more work in history -func (pa *ProcessorAdapter) CheckNoMoreWorkInHistory() bool { - return pa.processor.CheckNoMoreWorkInHistory() -} - -var testFormats = []string{ - "2006-01-02 15:04:05-07:00", - "2006-01-02 15:04:05+07:00", - "2006-01-02 15:04:05", -} - -// updateNewsgroupLastActivity updates newsgroups' updated_at field based on their latest article -func updateNewsgroupLastActivity(db *database.Database) error { - updatedCount := 0 - totalProcessed := 0 - var id int - var name string - var formattedDate string - var parsedDate time.Time - var latestDate sql.NullString - // Get newsgroups - rows, err := db.GetMainDB().Query("SELECT id, name FROM newsgroups WHERE message_count > 0") - if err != nil { - return fmt.Errorf("failed to query newsgroups: %w", err) - } - defer rows.Close() - for rows.Next() { - if err := rows.Scan(&id, &name); err != nil { - return fmt.Errorf("error [WEB]: updateNewsgroupLastActivity rows.Scan newsgroup: %v", err) - } - if err := updateNewsGroupActivityValue(db, &id, &name, &latestDate, &parsedDate, &formattedDate); err == nil { - updatedCount++ - } - totalProcessed++ - log.Printf("[WEB]: Processed %d newsgroups, updated %d so far", totalProcessed, updatedCount) - } - // Check for iteration errors - if err := rows.Err(); err != nil { - return fmt.Errorf("error updateNewsgroupLastActivity iterating newsgroups: %w", err) - } - - log.Printf("[WEB]: updateNewsgroupLastActivity completed: processed %d total newsgroups, updated %d", totalProcessed, updatedCount) - return nil -} - -const ActivityQuery = "UPDATE newsgroups SET updated_at = ? WHERE id = ? AND updated_at != ?" - -func updateNewsGroupActivityValue(db *database.Database, id *int, name *string, latestDate *sql.NullString, parsedDate *time.Time, formattedDate *string) error { - // Get the group database for this newsgroup - groupDBs, err := db.GetGroupDBs(*name) - if err != nil { - log.Printf("[WEB]: updateNewsgroupLastActivity GetGroupDB %s: %v", *name, err) - return err - } - - /* - _, err = database.RetryableExec(groupDBs.DB, "UPDATE articles SET spam = 1 WHERE spam = 0 AND hide = 1", nil) - if err != nil { - db.ForceCloseGroupDBs(groupDBs) - log.Printf("[WEB]: Failed to update spam flags for newsgroup %s: %v", name, err) - continue - } - */ - - // Query the latest article date from the group's articles table (excluding hidden articles) - rows, err := database.RetryableQuery(groupDBs.DB, "SELECT MAX(date_sent) FROM articles WHERE hide = 0 LIMIT 1", nil, latestDate) - //groupDBs.Return(db) // Always return the database connection - if err != nil { - log.Printf("[WEB]: updateNewsgroupLastActivity RetryableQueryRowScan %s: %v", *name, err) - return err - } - defer rows.Close() - defer db.ForceCloseGroupDBs(groupDBs) - for rows.Next() { - // Only update if we found a latest date - if latestDate.Valid { - // Parse the date and format it consistently as UTC - if latestDate.String == "" { - log.Printf("[WEB]: updateNewsgroupLastActivity empty latestDate.String in ng: '%s'", *name) - return fmt.Errorf("error updateNewsgroupLastActivity empty latestDate.String in ng: '%s'", *name) - } - // Try multiple date formats to handle various edge cases - for _, format := range testFormats { - *parsedDate, err = time.Parse(format, latestDate.String) - if err == nil { - break - } - } - if err != nil { - log.Printf("[WEB]: updateNewsgroupLastActivity parsing date '%s' for %s: %v", latestDate.String, *name, err) - return err - } - - // Format as UTC without timezone info to match db_batch.go format - *formattedDate = parsedDate.UTC().Format("2006-01-02 15:04:05") - result, err := db.GetMainDB().Exec(ActivityQuery, *formattedDate, *id, *formattedDate) - if err != nil { - log.Printf("[WEB]: error updateNewsgroupLastActivity updating newsgroup %s: %v", *name, err) - return err - } - if _, err := result.RowsAffected(); err != nil { - log.Printf("[WEB]: updateNewsgroupLastActivity: '%s' dateStr=%s formattedDate=%s", *name, latestDate.String, *formattedDate) - } - - } - } - return nil -} - -// hideFuturePosts updates articles' hide field to 1 if they are posted more than 48 hours in the future -func hideFuturePosts(db *database.Database) error { - // Calculate the cutoff time (current time + 48 hours) - cutoffTime := time.Now().Add(48 * time.Hour) - - // First, get all newsgroups from the main database - rows, err := db.GetMainDB().Query("SELECT id, name FROM newsgroups WHERE message_count > 0 AND active = 1") - if err != nil { - return fmt.Errorf("failed to query newsgroups: %w", err) - } - defer rows.Close() - - updatedArticles := 0 - processedGroups := 0 - skippedGroups := 0 - - for rows.Next() { - var id int - var name string - if err := rows.Scan(&id, &name); err != nil { - log.Printf("[WEB]: Future posts migration error scanning newsgroup: %v", err) - continue - } - - // Get the group database for this newsgroup - groupDBs, err := db.GetGroupDBs(name) - if err != nil { - log.Printf("[WEB]: Future posts migration error getting group DB for %s: %v", name, err) - skippedGroups++ - continue - } - - // Update articles that are posted more than 48 hours in the future - result, err := database.RetryableExec(groupDBs.DB, "UPDATE articles SET hide = 1, spam = 1 WHERE date_sent > ? AND hide = 0", cutoffTime.Format("2006-01-02 15:04:05")) - db.ForceCloseGroupDBs(groupDBs) - - if err != nil { - log.Printf("[WEB]: Future posts migration error updating articles for %s: %v", name, err) - skippedGroups++ - continue - } - - rowsAffected, err := result.RowsAffected() - if err != nil { - log.Printf("[WEB]: Future posts migration error getting rows affected for %s: %v", name, err) - skippedGroups++ - continue - } - - if rowsAffected > 0 { - log.Printf("[WEB]: Hidden %d future posts in newsgroup %s", rowsAffected, name) - updatedArticles += int(rowsAffected) - } - processedGroups++ - } - - if err := rows.Err(); err != nil { - return fmt.Errorf("error iterating newsgroups: %w", err) - } - - log.Printf("[WEB]: Future posts migration completed: processed %d groups, hidden %d articles, skipped %d groups", processedGroups, updatedArticles, skippedGroups) - return nil -} - -// writeActiveFileFromDB writes an NNTP active file from the main database newsgroups table -func writeActiveFileFromDB(db *database.Database, filePath string, activeOnly bool) error { - // Query newsgroups from main database with all fields needed for active file - active := 0 - if activeOnly { - active = 1 - } - rows, err := db.GetMainDB().Query(` - SELECT name, high_water, low_water, status - FROM newsgroups - WHERE active = ? - ORDER BY name - `, active) - if err != nil { - return fmt.Errorf("failed to query newsgroups: %w", err) - } - defer rows.Close() - - // Create the output file - file, err := os.Create(filePath) - if err != nil { - return fmt.Errorf("failed to create active file '%s': %w", filePath, err) - } - defer file.Close() - - totalGroups := 0 - log.Printf("[WEB]: Writing active file to: %s (writeActiveOnly=%t)", filePath, activeOnly) - - // Write each newsgroup in NNTP active file format: groupname highwater lowwater status - for rows.Next() { - var name string - var highWater, lowWater int64 - var status string - - if err := rows.Scan(&name, &highWater, &lowWater, &status); err != nil { - log.Printf("[WEB]: Warning: Failed to scan newsgroup row: %v", err) - continue - } - - // Validate status field (should be single character) - if len(status) != 1 { - log.Printf("[WEB]: Warning: Invalid status '%s' for group '%s', using 'y'", status, name) - status = "y" - } - - // Write in NNTP active file format: groupname highwater lowwater status - line := fmt.Sprintf("%s %d %d %s\n", name, highWater, lowWater, status) - if _, err := file.WriteString(line); err != nil { - return fmt.Errorf("failed to write line for group '%s': %w", name, err) - } - - totalGroups++ - if totalGroups%1000 == 0 { - log.Printf("[WEB]: Written %d groups to active file...", totalGroups) - } - } - - if err := rows.Err(); err != nil { - return fmt.Errorf("error iterating newsgroups: %w", err) - } - - log.Printf("[WEB]: Successfully wrote %d newsgroups to active file: %s", totalGroups, filePath) - return nil -} - -func rsyncInactiveGroupsToDir(db *database.Database, newdatadir string) error { - - // check if newdatadir exists - if _, err := os.Stat(newdatadir); os.IsNotExist(err) { - if err := os.MkdirAll(newdatadir, 0755); err != nil { - log.Printf("[WEB]: Warning: Failed to create new data directory '%s': %v", newdatadir, err) - return err - } - } - - rows, err := db.GetMainDB().Query(` - SELECT name - FROM newsgroups - WHERE active = 0 - ORDER BY name - `) - if err != nil { - return fmt.Errorf("failed to query newsgroups: %w", err) - } - defer rows.Close() - basedirNew := filepath.Join(newdatadir, "/db/") - for rows.Next() { - var newsgroup string - if err := rows.Scan(&newsgroup); err != nil { - log.Printf("[WEB]: Warning: Failed to scan newsgroup row: %v", err) - return err - } - groupsHash := database.MD5Hash(newsgroup) - baseGroupDBdir := filepath.Join("data", "/db/"+groupsHash) - baseGroupDBdirNew := filepath.Join(basedirNew, groupsHash) - sanitizedName := database.SanitizeGroupName(newsgroup) - groupDBfileOld := filepath.Join(baseGroupDBdir + "/" + sanitizedName + ".db") - if !database.FileExists(groupDBfileOld) { - log.Printf("[RSYNC]: Group database file does not exist: %s", groupDBfileOld) - continue - } - if _, err := os.Stat(baseGroupDBdirNew); os.IsNotExist(err) { - if err := os.MkdirAll(baseGroupDBdirNew, 0755); err != nil { - log.Printf("[WEB]: Warning: Failed to create new data directory '%s': %v", newdatadir, err) - return err - } - } - start := time.Now() - if err := database.RsyncDIR(baseGroupDBdir, basedirNew, rsyncRemoveSource); err != nil { - log.Printf("[RSYNC]: Warning: Failed to rsync group database file baseGroupDBdir='%s' to basedirNew='%s' (baseGroupDBdirNew=%s): %v", baseGroupDBdir, basedirNew, baseGroupDBdirNew, err) - return err - } - groupDBfileNew := filepath.Join(baseGroupDBdirNew + "/" + sanitizedName + ".db") - if !database.FileExists(groupDBfileNew) { - log.Printf("[RSYNC]: ERROR: new group database file not found: %s", groupDBfileNew) - return fmt.Errorf("error new group database file not found: %s", groupDBfileNew) - } - log.Printf("[RSYNC]: OK %s (%v) '%s' to '%s'", newsgroup, time.Since(start), baseGroupDBdir, baseGroupDBdirNew) - } - return nil -} - var appVersion = "-unset-" func main() { @@ -412,7 +94,7 @@ func main() { //flag.BoolVar(&withfetch, "withfetch", false, "Enable internal Cronjob to fetch new articles") //flag.Int64Var(&isleep, "isleep", 300, "Sleeps in fetch routines. if started with: -withfetch (default: 300 seconds = 5min)") //flag.Int64Var(&ignoreInitialTinyGroups, "ignore-initial-tiny-groups", 0, "If > 0: initial fetch ignores tiny groups with fewer articles than this (default: 0)") - flag.StringVar(&hostnamePath, "nntphostname", "", "your hostname must be set") + flag.StringVar(&nntphostname, "nntphostname", "", "your hostname must be set") flag.StringVar(&webcertFile, "websslcert", "", "SSL certificate file (/path/to/fullchain.pem)") flag.StringVar(&webkeyFile, "websslkey", "", "SSL key file (/path/to/privkey.pem)") flag.IntVar(&nntptcpport, "nntptcpport", 0, "NNTP TCP port") @@ -430,6 +112,8 @@ func main() { flag.BoolVar(&writeActiveOnly, "write-active-only", true, "use with -write-active-file (false writes only non active groups!)") flag.StringVar(&rsyncInactiveGroups, "rsync-inactive-groups", "", "path to new data dir, uses rsync to copy all inactive group databases to new data folder.") flag.BoolVar(&rsyncRemoveSource, "rsync-remove-source", false, "use with -rsync-inactive-groups. if set, removes source files after moving inactive groups (default: false)") + flag.StringVar(&compareActiveFile, "compare-active", "", "Compare active file with database and show missing groups (format: groupname highwater lowwater status)") + flag.Int64Var(&compareActiveMinArticles, "compare-active-min-articles", 0, "use with -compare-active: only show groups with more than N articles (calculated as high-low)") /* flag.BoolVar(&enableFediverse, "enable-fediverse", false, "Enable Fediverse bridge (default: false)") flag.StringVar(&fediverseDomain, "fediverse-domain", "", "Fediverse domain (e.g. example.com)") @@ -481,48 +165,11 @@ func main() { } log.Printf("[WEB]: Using WEB configuration: %#v", webConfig) - // Override config with command-line flags if provided - if withnntp && nntptcpport > 0 { - mainConfig.Server.NNTP.Port = nntptcpport - log.Printf("[WEB]: Overriding NNTP TCP port with command-line flag: %d", mainConfig.Server.NNTP.Port) - } else { - log.Printf("[WEB]: No NNTP TCP port flag provided") - mainConfig.Server.NNTP.Port = 0 - } - if withnntp && nntptlsport > 0 { - mainConfig.Server.NNTP.TLSPort = nntptlsport - mainConfig.Server.NNTP.TLSCert = nntpcertFile - mainConfig.Server.NNTP.TLSKey = nntpkeyFile - } else { - mainConfig.Server.NNTP.TLSPort = 0 - mainConfig.Server.NNTP.TLSCert = "" - mainConfig.Server.NNTP.TLSKey = "" - log.Printf("[WEB]: No NNTP TLS port flag provided") - } - - if hostnamePath == "" && (withfetch || withnntp) { - log.Fatalf("[WEB]: Error: hostname must be set when starting with -withfetch or -withnntp") - } - mainConfig.Server.Hostname = hostnamePath - processor.LocalHostnamePath = hostnamePath - log.Printf("[WEB]: Using NNTP configuration %#v", mainConfig.Server.NNTP) - // Validate port if webConfig.ListenPort < 1024 || webConfig.ListenPort > 65535 { log.Fatalf("[WEB]: Invalid port number: %d (must be between 1024 and 65535)", webConfig.ListenPort) } - // Validate port - if mainConfig.Server.NNTP.Port > 0 { - if mainConfig.Server.NNTP.Port < 1024 || mainConfig.Server.NNTP.Port > 65535 { - log.Fatalf("[WEB]: Invalid NNTP tcp port number: %d (must be between 1024 and 65535)", mainConfig.Server.NNTP.Port) - } - } - // Validate port - if mainConfig.Server.NNTP.TLSPort > 0 { - if mainConfig.Server.NNTP.TLSPort < 1024 || mainConfig.Server.NNTP.TLSPort > 65535 { - log.Fatalf("[WEB]: Invalid NNTP tls port number: %d (must be between 1024 and 65535)", mainConfig.Server.NNTP.TLSPort) - } - } + /* // Check for environment variable override if portEnv := os.Getenv("PUGLEAF_WEB_PORT"); portEnv != "" { @@ -566,6 +213,7 @@ func main() { // Note: Database batch workers are started automatically by OpenDatabase() db.WG.Add(2) // Adds to wait group for db_batch.go cron jobs db.WG.Add(1) // Adds for history: one for writer worker + db.WG.Add(1) // Adds for processor // Apply main database migrations if err := db.Migrate(); err != nil { @@ -573,6 +221,11 @@ func main() { } //log.Printf("[WEB]: Database migrations applied successfully") + // Set hostname in processor with database fallback support + log.Printf("nntphostname=%s", nntphostname) + if err := processor.SetHostname(nntphostname, db); err != nil { + log.Fatalf("[WEB]: Failed to set NNTP hostname: %v", err) + } // Run future posts hiding migration first if requested if updateNewsgroupsHideFuture { log.Printf("[WEB]: Starting future posts hiding migration...") @@ -623,6 +276,18 @@ func main() { } } + // compareActiveFile + if compareActiveFile != "" { + log.Printf("[WEB]: Comparing active file with database: %s (min articles: %d)", compareActiveFile, compareActiveMinArticles) + if err := compareActiveFileWithDatabase(db, compareActiveFile, compareActiveMinArticles); err != nil { + log.Printf("[WEB]: Error: Failed to compare active file: %v", err) + os.Exit(1) + } else { + log.Printf("[WEB]: Active file comparison completed successfully") + os.Exit(0) + } + } + // Get or set history UseShortHashLen configuration finalUseShortHashLen, isLocked, err := db.GetHistoryUseShortHashLen(useShortHashLen) if err != nil { @@ -705,23 +370,48 @@ func main() { log.Printf("[WEB]: Found %d newsgroups in database", len(groups)) } - // Only create processor if integrated fetcher or nntp-server is enabled - var proc *processor.Processor - if withfetch || withnntp { - proc = NewFetchProcessor(db) // Create a new processor instance - if proc == nil { - log.Printf("[WEB]: ERROR: No enabled providers found! Cannot proceed with article fetching") - } else { - log.Printf("[WEB]: Using first enabled provider for fetching articles") - } + proc := NewFetchProcessor(db) + if proc == nil { + log.Fatalf("[WEB]: Error booting processor") } - var nntpServer *nntp.NNTPServer if (nntptcpport > 0 || nntptlsport > 0) && withnntp { - if proc == nil { - log.Fatalf("[WEB]: Cannot start NNTP server without a processor (no enabled providers found)") + // set config from command-line flags + if nntptcpport > 0 { + mainConfig.Server.NNTP.Port = nntptcpport + log.Printf("[WEB]: Local NNTP Server TCP: %d", nntptcpport) + } else { + log.Printf("[WEB]: No NNTP TCP port flag provided") + mainConfig.Server.NNTP.Port = 0 + } + if nntptlsport > 0 { + log.Printf("[WEB]: Local NNTP Server TLS: %d", nntptlsport) + mainConfig.Server.NNTP.TLSPort = nntptlsport + mainConfig.Server.NNTP.TLSCert = nntpcertFile + mainConfig.Server.NNTP.TLSKey = nntpkeyFile + } else { + log.Printf("[WEB]: No NNTP TLS port flag provided") + mainConfig.Server.NNTP.TLSPort = 0 + mainConfig.Server.NNTP.TLSCert = "" + mainConfig.Server.NNTP.TLSKey = "" + } + mainConfig.Server.Hostname = nntphostname + // Validate port + if mainConfig.Server.NNTP.Port > 0 { + if mainConfig.Server.NNTP.Port < 1024 || mainConfig.Server.NNTP.Port > 65535 { + log.Fatalf("[WEB]: Invalid NNTP tcp port number: %d (must be between 1024 and 65535)", mainConfig.Server.NNTP.Port) + } + } + // Validate port + if mainConfig.Server.NNTP.TLSPort > 0 { + if mainConfig.Server.NNTP.TLSPort < 1024 || mainConfig.Server.NNTP.TLSPort > 65535 { + log.Fatalf("[WEB]: Invalid NNTP tls port number: %d (must be between 1024 and 65535)", mainConfig.Server.NNTP.TLSPort) + } } processorAdapter := NewProcessorAdapter(proc) + if processorAdapter == nil { + log.Fatalf("[WEB]: Failed to create processor adapter for NNTP server") + } log.Printf("[WEB]: Starting NNTP server with TCP port %d, TLS port %d", mainConfig.Server.NNTP.Port, mainConfig.Server.NNTP.TLSPort) nntpServer, err = nntp.NewNNTPServer(db, &mainConfig.Server, db.WG, processorAdapter) if err != nil { @@ -738,6 +428,13 @@ func main() { go FetchRoutine(db, proc, finalUseShortHashLen, true, isleep, DLParChan, progressDB) // Start the processor routine in a separate goroutine } + var postQueueWorker *processor.PostQueueWorker + if proc != nil { + postQueueWorker = proc.NewPostQueueWorker() + postQueueWorker.Start() + log.Printf("[WEB]: PostQueueWorker started for web posting") + } + // Create and start web server in a goroutine for non-blocking startup server := web.NewServer(db, webConfig, nntpServer) @@ -781,6 +478,14 @@ func main() { log.Printf("[WEB]: NNTP server stopped successfully") } } + + // Stop PostQueueWorker if running + if postQueueWorker != nil { + log.Printf("[WEB]: Stopping PostQueueWorker...") + postQueueWorker.Stop() + log.Printf("[WEB]: PostQueueWorker stopped") + } + // Signal background tasks to stop close(db.StopChan) @@ -811,305 +516,3 @@ func main() { log.Printf("[WEB]: Graceful shutdown completed") } // end main - -// startHierarchyUpdater runs a background job every N minutes to update -// hierarchy last_updated fields based on their child newsgroups -func startHierarchyUpdater(db *database.Database) { - // Run immediately on startup - if err := db.UpdateHierarchiesLastUpdated(); err != nil { - log.Printf("[WEB]: Initial hierarchy update failed: %v", err) - } else { - // Update the hierarchy cache with new last_updated values - if db.HierarchyCache != nil { - if err := db.HierarchyCache.UpdateHierarchyLastUpdated(db); err != nil { - log.Printf("[WEB]: Initial hierarchy cache update failed: %v", err) - } else { - log.Printf("[WEB]: Initial hierarchy cache updated successfully") - } - } - } - log.Printf("[WEB]: Hierarchy updater started, will sync hierarchy last_updated every 30 minutes") - - for { - time.Sleep(10 * time.Minute) - if err := db.UpdateHierarchiesLastUpdated(); err != nil { - log.Printf("[WEB]: Hierarchy update failed: %v", err) - } else { - // Update the hierarchy cache with new last_updated values - if db.HierarchyCache != nil { - if err := db.HierarchyCache.UpdateHierarchyLastUpdated(db); err != nil { - log.Printf("[WEB]: Hierarchy cache update failed: %v", err) - } else { - log.Printf("[WEB]: Hierarchy cache updated successfully") - } - } - } - } -} - -// monitorUpdateFile checks for the existence of an .update file every 30 seconds -// and signals for shutdown when found, then removes the file -func monitorUpdateFile(shutdownChan chan<- bool) { - updateFilePath := ".update" - ticker := time.NewTicker(60 * time.Second) - defer ticker.Stop() - - log.Printf("[WEB]: Update file monitor started, checking for '%s' every 60 seconds", updateFilePath) - - for range ticker.C { - - // Check if .update file exists - if _, err := os.Stat(updateFilePath); err == nil { - log.Printf("[WEB]: Update file '%s' detected, triggering graceful shutdown", updateFilePath) - - // Rename the update file - if err := os.Rename(updateFilePath, updateFilePath+".todo"); err != nil { - log.Printf("[WEB]: Warning: Failed to rename update file '%s': %v", updateFilePath, err) - continue - } else { - log.Printf("[WEB]: Update file '%s' renamed successfully", updateFilePath) - } - - // Signal shutdown - select { - case shutdownChan <- true: - log.Printf("[WEB]: Shutdown signal sent via update file monitor") - default: - log.Printf("[WEB]: Shutdown channel already signaled") - } - return - } - // File doesn't exist, continue monitoring - } -} - -func ConnectPools(db *database.Database) []*nntp.Pool { - log.Printf("[WEB]: Fetching providers from database...") - providers, err := db.GetProviders() - if err != nil { - log.Fatalf("[WEB]: Failed to fetch providers: %v", err) - } - log.Printf("[WEB]: Found %d providers in database", len(providers)) - pools := make([]*nntp.Pool, 0, len(providers)) - log.Printf("[WEB]: Create provider connection pools...") - enabledProviders := 0 - for _, p := range providers { - if !p.Enabled || p.Host == "" || p.Port <= 0 { - log.Printf("[WEB]: Skipping disabled/invalid provider: %s (ID: %d, Enabled: %v, Host: '%s', Port: %d)", - p.Name, p.ID, p.Enabled, p.Host, p.Port) - continue // Skip disabled providers - } - enabledProviders++ - // Convert models.Provider to config.Provider for the BackendConfig - configProvider := &config.Provider{ - Grp: p.Grp, - Name: p.Name, - Host: p.Host, - Port: p.Port, - SSL: p.SSL, - Username: p.Username, - Password: p.Password, - MaxConns: p.MaxConns, - Enabled: p.Enabled, - Priority: p.Priority, - MaxArtSize: p.MaxArtSize, - } - - backendConfig := &nntp.BackendConfig{ - Host: p.Host, - Port: p.Port, - SSL: p.SSL, - Username: p.Username, - Password: p.Password, - //ConnectTimeout: 9 * time.Second, - //ReadTimeout: 60 * time.Second, - //WriteTimeout: 60 * time.Second, - MaxConns: p.MaxConns, - Provider: configProvider, // Set the Provider field - } - - pool := nntp.NewPool(backendConfig) - pool.StartCleanupWorker(5 * time.Second) - pools = append(pools, pool) - log.Printf("[WEB]: Using only first enabled provider '%s' (TODO: support multiple providers)", p.Name) - break // For now, we only use the first enabled provider!!! TODO - } - log.Printf("[WEB]: %d providers, %d enabled, using %d pools", len(providers), enabledProviders, len(pools)) - return pools -} - -func NewFetchProcessor(db *database.Database) *processor.Processor { - var pool *nntp.Pool - pools := ConnectPools(db) // Get NNTP pools - if len(pools) == 0 { - log.Printf("[WEB]: ERROR: No enabled providers found! Cannot proceed with article fetching") - pool = nil - } else { - pool = pools[0] - } - log.Printf("[WEB]: Creating processor instance with useShortHashLen=%d...", useShortHashLen) - proc := processor.NewProcessor(db, pool, useShortHashLen) // Create a new processor instance - log.Printf("[WEB]: Processor created successfully") - return proc -} - -func FetchRoutine(db *database.Database, proc *processor.Processor, useShortHashLen int, boot bool, isleep int64, DLParChan chan struct{}, progressDB *database.ProgressDB) { - /* DISABLED - if isleep < 15 { - isleep = 15 // min 15 sec sleep! - } - startTime := time.Now() - log.Printf("[WEB]: FetchRoutine STARTED (boot=%v, useShortHashLen=%d) at %v", boot, useShortHashLen, startTime) - - webmutex.Lock() - log.Printf("[WEB]: Acquired webmutex lock") - defer func() { - webmutex.Unlock() - log.Printf("[WEB]: Released webmutex lock") - }() - - if boot { - log.Printf("[WEB]: Boot mode detected - waiting %v before starting...", isleep) - select { - case <-db.StopChan: - log.Printf("[WEB]: Shutdown detected during boot wait, exiting FetchRoutine") - return - case <-time.After(time.Duration(isleep) * time.Second): - log.Printf("[WEB]: Boot wait completed, starting article fetching") - } - } - log.Printf("[WEB]: Begin article fetching process") - - defer func(isleep int64, progressDB *database.ProgressDB) { - duration := time.Since(startTime) - log.Printf("[WEB]: FetchRoutine COMPLETED after %v", duration) - if isleep > 30 { - proc.Pool.ClosePool() // Close the NNTP pool if sleep is more than 30 seconds - } - // Check if shutdown was requested before scheduling restart - select { - case <-db.StopChan: - log.Printf("[WEB]: Shutdown detected, not restarting FetchRoutine") - return - default: - // pass - } - // Sleep with shutdown check - wait: - for { - select { - case <-db.StopChan: - log.Printf("[WEB]: Shutdown detected during sleep, not restarting FetchRoutine") - return - case <-time.After(time.Duration(isleep) * time.Second): - break wait - } - } - pools := ConnectPools(db) // Reconnect pools after sleep - if len(pools) == 0 { - log.Printf("[WEB]: ERROR: No enabled providers found after sleep! Cannot proceed with article fetching") - } else { - proc.Pool = pools[0] // Use the first pool - } - log.Printf("[WEB]: Sleep completed, starting new FetchRoutine goroutine") - go FetchRoutine(db, proc, useShortHashLen, false, isleep, DLParChan, progressDB) - log.Printf("[WEB]: New FetchRoutine goroutine launched") - }(isleep, progressDB) - - log.Printf("[WEB]: Fetching newsgroups from database...") - groups, err := db.MainDBGetAllNewsgroups() - if err != nil { - log.Printf("[WEB]: FATAL: Could not fetch newsgroups: %v", err) - log.Printf("[WEB]: Warning: Could not fetch newsgroups: %v", err) - return - } - - log.Printf("[WEB]: Found %d newsgroups in database", len(groups)) - - // Check for data integrity issues - emptyNameCount := 0 - for _, group := range groups { - if group.Name == "" { - emptyNameCount++ - log.Printf("[WEB]: WARNING: Found newsgroup with empty name (ID: %d)", group.ID) - } - } - if emptyNameCount > 0 { - log.Printf("[WEB]: WARNING: Found %d newsgroups with empty names - this indicates a data integrity issue", emptyNameCount) - } - - // Configure bridges if enabled - if enableFediverse || enableMatrix { - bridgeConfig := &processor.BridgeConfig{ - FediverseEnabled: enableFediverse, - FediverseDomain: fediverseDomain, - FediverseBaseURL: fediverseBaseURL, - MatrixEnabled: enableMatrix, - MatrixHomeserver: matrixHomeserver, - MatrixAccessToken: matrixAccessToken, - MatrixUserID: matrixUserID, - } - proc.EnableBridges(bridgeConfig) - log.Printf("[WEB]: Bridge configuration applied") - } else { - log.Printf("[WEB]: No bridges enabled (use --enable-fediverse or --enable-matrix to enable)") - } - - log.Printf("[WEB]: Starting to process %d newsgroups...", len(groups)) - processedCount := 0 - upToDateCount := 0 - errorCount := 0 - - for i, group := range groups { - // Check for shutdown before processing each group - select { - case <-db.StopChan: - log.Printf("[WEB]: Shutdown detected, stopping newsgroup processing at group %d/%d", i+1, len(groups)) - return - default: - // Continue processing - } - - // Skip groups with empty names - if group.Name == "" { - log.Printf("[WEB]: [%d/%d] Skipping group with empty name (ID: %d)", i+1, len(groups), group.ID) - errorCount++ - continue - } - - log.Printf("[WEB]: [%d/%d] Processing group: %s (ID: %d)", i+1, len(groups), group.Name, group.ID) - - // Register newsgroup with bridges if enabled - if proc.BridgeManager != nil { - if err := proc.BridgeManager.RegisterNewsgroup(group); err != nil { - log.Printf("web.main.go: [%d/%d] Warning: Failed to register group %s with bridges: %v", i+1, len(groups), group.Name, err) - } - } - - // Check if the group is in sections DB - err := proc.DownloadArticles(group.Name, ignoreInitialTinyGroups, DLParChan, progressDB, 0 ,0) - if err != nil { - if err.Error() == "up2date" { - log.Printf("[WEB]: [%d/%d] Group %s is up to date, skipping", i+1, len(groups), group.Name) - upToDateCount++ - } else { - log.Printf("[WEB]: [%d/%d] ERROR processing group %s: %v", i+1, len(groups), group.Name, err) - errorCount++ - - // If we're getting too many consecutive errors, it might be an auth issue - if errorCount > 10 && (processedCount+upToDateCount) == 0 { - log.Printf("[WEB]: WARNING: Too many consecutive errors (%d), this might indicate authentication or connection issues", errorCount) - } - } - } else { - log.Printf("[WEB]: [%d/%d] Successfully processed group %s", i+1, len(groups), group.Name) - processedCount++ - } - log.Printf("[WEB]: Progress: processed=%d, up-to-date=%d, errors=%d, remaining=%d", - processedCount, upToDateCount, errorCount, len(groups)-(i+1)) - } - log.Printf("[WEB]: Finished processing all %d groups", len(groups)) - log.Printf("[WEB]: FINAL SUMMARY - Total groups: %d, Successfully processed: %d, Up-to-date: %d, Errors: %d", - len(groups), processedCount, upToDateCount, errorCount) - */ -} diff --git a/cmd/web/main_adapters.go b/cmd/web/main_adapters.go new file mode 100644 index 00000000..61b4ad9d --- /dev/null +++ b/cmd/web/main_adapters.go @@ -0,0 +1,34 @@ +package main + +import ( + "github.com/go-while/go-pugleaf/internal/history" + "github.com/go-while/go-pugleaf/internal/models" + "github.com/go-while/go-pugleaf/internal/processor" +) + +// ProcessorAdapter adapts the processor.Processor to implement nntp.ArticleProcessor interface +type ProcessorAdapter struct { + processor *processor.Processor +} + +// NewProcessorAdapter creates a new processor adapter +func NewProcessorAdapter(proc *processor.Processor) *ProcessorAdapter { + return &ProcessorAdapter{processor: proc} +} + +// ProcessIncomingArticle processes an incoming article +func (pa *ProcessorAdapter) ProcessIncomingArticle(article *models.Article) (int, error) { + // Forward the Article directly to the processor + // No conversions needed since both use models.Article + return pa.processor.ProcessIncomingArticle(article) +} + +// Lookup checks if a message-ID exists in history +func (pa *ProcessorAdapter) Lookup(msgIdItem *history.MessageIdItem) (int, error) { + return pa.processor.History.Lookup(msgIdItem) +} + +// CheckNoMoreWorkInHistory checks if there's no more work in history +func (pa *ProcessorAdapter) CheckNoMoreWorkInHistory() bool { + return pa.processor.CheckNoMoreWorkInHistory() +} diff --git a/cmd/web/main_functions.go b/cmd/web/main_functions.go new file mode 100644 index 00000000..7f24b3cb --- /dev/null +++ b/cmd/web/main_functions.go @@ -0,0 +1,754 @@ +package main + +import ( + "bufio" + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/go-while/go-pugleaf/internal/config" + "github.com/go-while/go-pugleaf/internal/database" + "github.com/go-while/go-pugleaf/internal/models" + "github.com/go-while/go-pugleaf/internal/nntp" + "github.com/go-while/go-pugleaf/internal/processor" +) + +var testFormats = []string{ + "2006-01-02 15:04:05-07:00", + "2006-01-02 15:04:05+07:00", + "2006-01-02 15:04:05", +} + +// updateNewsgroupLastActivity updates newsgroups' updated_at field based on their latest article +func updateNewsgroupLastActivity(db *database.Database) error { + updatedCount := 0 + totalProcessed := 0 + var id int + var name string + var formattedDate string + var parsedDate time.Time + var latestDate sql.NullString + // Get newsgroups + rows, err := db.GetMainDB().Query("SELECT id, name FROM newsgroups WHERE message_count > 0") + if err != nil { + return fmt.Errorf("failed to query newsgroups: %w", err) + } + defer rows.Close() + for rows.Next() { + if err := rows.Scan(&id, &name); err != nil { + return fmt.Errorf("error [WEB]: updateNewsgroupLastActivity rows.Scan newsgroup: %v", err) + } + if err := updateNewsGroupActivityValue(db, &id, &name, &latestDate, &parsedDate, &formattedDate); err == nil { + updatedCount++ + } + totalProcessed++ + log.Printf("[WEB]: Processed %d newsgroups, updated %d so far", totalProcessed, updatedCount) + } + // Check for iteration errors + if err := rows.Err(); err != nil { + return fmt.Errorf("error updateNewsgroupLastActivity iterating newsgroups: %w", err) + } + + log.Printf("[WEB]: updateNewsgroupLastActivity completed: processed %d total newsgroups, updated %d", totalProcessed, updatedCount) + return nil +} + +const ActivityQuery = "UPDATE newsgroups SET updated_at = ? WHERE id = ? AND updated_at != ?" + +func updateNewsGroupActivityValue(db *database.Database, id *int, name *string, latestDate *sql.NullString, parsedDate *time.Time, formattedDate *string) error { + // Get the group database for this newsgroup + groupDBs, err := db.GetGroupDBs(*name) + if err != nil { + log.Printf("[WEB]: updateNewsgroupLastActivity GetGroupDB %s: %v", *name, err) + return err + } + + /* + _, err = database.RetryableExec(groupDBs.DB, "UPDATE articles SET spam = 1 WHERE spam = 0 AND hide = 1", nil) + if err != nil { + db.ForceCloseGroupDBs(groupDBs) + log.Printf("[WEB]: Failed to update spam flags for newsgroup %s: %v", name, err) + continue + } + */ + + // Query the latest article date from the group's articles table (excluding hidden articles) + rows, err := database.RetryableQuery(groupDBs.DB, "SELECT MAX(date_sent) FROM articles WHERE hide = 0 LIMIT 1", nil, latestDate) + //groupDBs.Return(db) // Always return the database connection + if err != nil { + log.Printf("[WEB]: updateNewsgroupLastActivity RetryableQueryRowScan %s: %v", *name, err) + return err + } + defer rows.Close() + defer db.ForceCloseGroupDBs(groupDBs) + for rows.Next() { + // Only update if we found a latest date + if latestDate.Valid { + // Parse the date and format it consistently as UTC + if latestDate.String == "" { + log.Printf("[WEB]: updateNewsgroupLastActivity empty latestDate.String in ng: '%s'", *name) + return fmt.Errorf("error updateNewsgroupLastActivity empty latestDate.String in ng: '%s'", *name) + } + // Try multiple date formats to handle various edge cases + for _, format := range testFormats { + *parsedDate, err = time.Parse(format, latestDate.String) + if err == nil { + break + } + } + if err != nil { + log.Printf("[WEB]: updateNewsgroupLastActivity parsing date '%s' for %s: %v", latestDate.String, *name, err) + return err + } + + // Format as UTC without timezone info to match db_batch.go format + *formattedDate = parsedDate.UTC().Format("2006-01-02 15:04:05") + result, err := db.GetMainDB().Exec(ActivityQuery, *formattedDate, *id, *formattedDate) + if err != nil { + log.Printf("[WEB]: error updateNewsgroupLastActivity updating newsgroup %s: %v", *name, err) + return err + } + if _, err := result.RowsAffected(); err != nil { + log.Printf("[WEB]: updateNewsgroupLastActivity: '%s' dateStr=%s formattedDate=%s", *name, latestDate.String, *formattedDate) + } + + } + } + return nil +} + +// hideFuturePosts updates articles' hide field to 1 if they are posted more than 48 hours in the future +func hideFuturePosts(db *database.Database) error { + // Calculate the cutoff time (current time + 48 hours) + cutoffTime := time.Now().Add(48 * time.Hour) + + // First, get all newsgroups from the main database + rows, err := db.GetMainDB().Query("SELECT id, name FROM newsgroups WHERE message_count > 0 AND active = 1") + if err != nil { + return fmt.Errorf("failed to query newsgroups: %w", err) + } + defer rows.Close() + + updatedArticles := 0 + processedGroups := 0 + skippedGroups := 0 + + for rows.Next() { + var id int + var name string + if err := rows.Scan(&id, &name); err != nil { + log.Printf("[WEB]: Future posts migration error scanning newsgroup: %v", err) + continue + } + + // Get the group database for this newsgroup + groupDBs, err := db.GetGroupDBs(name) + if err != nil { + log.Printf("[WEB]: Future posts migration error getting group DB for %s: %v", name, err) + skippedGroups++ + continue + } + + // Update articles that are posted more than 48 hours in the future + result, err := database.RetryableExec(groupDBs.DB, "UPDATE articles SET hide = 1, spam = 1 WHERE date_sent > ? AND hide = 0", cutoffTime.Format("2006-01-02 15:04:05")) + db.ForceCloseGroupDBs(groupDBs) + + if err != nil { + log.Printf("[WEB]: Future posts migration error updating articles for %s: %v", name, err) + skippedGroups++ + continue + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + log.Printf("[WEB]: Future posts migration error getting rows affected for %s: %v", name, err) + skippedGroups++ + continue + } + + if rowsAffected > 0 { + log.Printf("[WEB]: Hidden %d future posts in newsgroup %s", rowsAffected, name) + updatedArticles += int(rowsAffected) + } + processedGroups++ + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("error iterating newsgroups: %w", err) + } + + log.Printf("[WEB]: Future posts migration completed: processed %d groups, hidden %d articles, skipped %d groups", processedGroups, updatedArticles, skippedGroups) + return nil +} + +// writeActiveFileFromDB writes an NNTP active file from the main database newsgroups table +func writeActiveFileFromDB(db *database.Database, filePath string, activeOnly bool) error { + // Query newsgroups from main database with all fields needed for active file + active := 0 + if activeOnly { + active = 1 + } + rows, err := db.GetMainDB().Query(` + SELECT name, high_water, low_water, status + FROM newsgroups + WHERE active = ? + ORDER BY name + `, active) + if err != nil { + return fmt.Errorf("failed to query newsgroups: %w", err) + } + defer rows.Close() + + // Create the output file + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create active file '%s': %w", filePath, err) + } + defer file.Close() + + totalGroups := 0 + log.Printf("[WEB]: Writing active file to: %s (writeActiveOnly=%t)", filePath, activeOnly) + + // Write each newsgroup in NNTP active file format: groupname highwater lowwater status + for rows.Next() { + var name string + var highWater, lowWater int64 + var status string + + if err := rows.Scan(&name, &highWater, &lowWater, &status); err != nil { + log.Printf("[WEB]: Warning: Failed to scan newsgroup row: %v", err) + continue + } + + // Validate status field (should be single character) + if len(status) != 1 { + log.Printf("[WEB]: Warning: Invalid status '%s' for group '%s', using 'y'", status, name) + status = "y" + } + + // Write in NNTP active file format: groupname highwater lowwater status + line := fmt.Sprintf("%s %d %d %s\n", name, highWater, lowWater, status) + if _, err := file.WriteString(line); err != nil { + return fmt.Errorf("failed to write line for group '%s': %w", name, err) + } + + totalGroups++ + if totalGroups%1000 == 0 { + log.Printf("[WEB]: Written %d groups to active file...", totalGroups) + } + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("error iterating newsgroups: %w", err) + } + + log.Printf("[WEB]: Successfully wrote %d newsgroups to active file: %s", totalGroups, filePath) + return nil +} + +func rsyncInactiveGroupsToDir(db *database.Database, newdatadir string) error { + + // check if newdatadir exists + if _, err := os.Stat(newdatadir); os.IsNotExist(err) { + if err := os.MkdirAll(newdatadir, 0755); err != nil { + log.Printf("[WEB]: Warning: Failed to create new data directory '%s': %v", newdatadir, err) + return err + } + } + + rows, err := db.GetMainDB().Query(` + SELECT name + FROM newsgroups + WHERE active = 0 + ORDER BY name + `) + if err != nil { + return fmt.Errorf("failed to query newsgroups: %w", err) + } + defer rows.Close() + basedirNew := filepath.Join(newdatadir, "/db/") + for rows.Next() { + var newsgroup string + if err := rows.Scan(&newsgroup); err != nil { + log.Printf("[WEB]: Warning: Failed to scan newsgroup row: %v", err) + return err + } + groupsHash := database.MD5Hash(newsgroup) + baseGroupDBdir := filepath.Join("data", "/db/"+groupsHash) + baseGroupDBdirNew := filepath.Join(basedirNew, groupsHash) + sanitizedName := database.SanitizeGroupName(newsgroup) + groupDBfileOld := filepath.Join(baseGroupDBdir + "/" + sanitizedName + ".db") + if !database.FileExists(groupDBfileOld) { + log.Printf("[RSYNC]: Group database file does not exist: %s", groupDBfileOld) + continue + } + if _, err := os.Stat(baseGroupDBdirNew); os.IsNotExist(err) { + if err := os.MkdirAll(baseGroupDBdirNew, 0755); err != nil { + log.Printf("[WEB]: Warning: Failed to create new data directory '%s': %v", newdatadir, err) + return err + } + } + start := time.Now() + if err := database.RsyncDIR(baseGroupDBdir, basedirNew, rsyncRemoveSource); err != nil { + log.Printf("[RSYNC]: Warning: Failed to rsync group database file baseGroupDBdir='%s' to basedirNew='%s' (baseGroupDBdirNew=%s): %v", baseGroupDBdir, basedirNew, baseGroupDBdirNew, err) + return err + } + groupDBfileNew := filepath.Join(baseGroupDBdirNew + "/" + sanitizedName + ".db") + if !database.FileExists(groupDBfileNew) { + log.Printf("[RSYNC]: ERROR: new group database file not found: %s", groupDBfileNew) + return fmt.Errorf("error new group database file not found: %s", groupDBfileNew) + } + log.Printf("[RSYNC]: OK %s (%v) '%s' to '%s'", newsgroup, time.Since(start), baseGroupDBdir, baseGroupDBdirNew) + } + return nil +} + +// compareActiveFileWithDatabase compares groups from active file with database and shows missing groups +func compareActiveFileWithDatabase(db *database.Database, activeFilePath string, minArticles int64) error { + log.Printf("[WEB]: Comparing active file '%s' with database (min articles: %d)...", activeFilePath, minArticles) + + // Open and read the active file + file, err := os.Open(activeFilePath) + if err != nil { + return fmt.Errorf("failed to open active file '%s': %w", activeFilePath, err) + } + defer file.Close() + + // Get all newsgroups from database for comparison + dbGroups, err := db.MainDBGetAllNewsgroups() + if err != nil { + return fmt.Errorf("failed to get newsgroups from database: %w", err) + } + + // Create a map for fast lookup of existing groups + existingGroups := make(map[string]*models.Newsgroup) + for _, group := range dbGroups { + existingGroups[group.Name] = group + } + + log.Printf("[WEB]: Found %d newsgroups in database for comparison", len(dbGroups)) + + // Parse active file and find missing groups + scanner := bufio.NewScanner(file) + lineNum := 0 + totalGroupsInFile := 0 + filteredGroups := 0 + missingGroups := 0 + var missingGroupsList []string + + fmt.Printf("\n=== Comparing Active File with Database ===\n") + fmt.Printf("Active file: %s\n", activeFilePath) + fmt.Printf("Min articles filter: %d\n", minArticles) + fmt.Printf("Database groups: %d\n\n", len(dbGroups)) + + for scanner.Scan() { + lineNum++ + line := strings.TrimSpace(scanner.Text()) + + // Skip empty lines and comments + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Parse active file format: groupname high low status + fields := strings.Fields(line) + if len(fields) < 4 { + log.Printf("[WEB]: Warning: Skipping malformed line %d in active file: %s", lineNum, line) + continue + } + + groupName := fields[0] + highWaterStr := fields[1] + lowWaterStr := fields[2] + status := fields[3] + + // Skip if group name is empty + if groupName == "" { + continue + } + + totalGroupsInFile++ + + // Parse high/low water marks to calculate article count + highWater, err := strconv.ParseInt(highWaterStr, 10, 64) + if err != nil { + log.Printf("[WEB]: Warning: Invalid high water mark '%s' at line %d for group %s", highWaterStr, lineNum, groupName) + continue + } + + lowWater, err := strconv.ParseInt(lowWaterStr, 10, 64) + if err != nil { + log.Printf("[WEB]: Warning: Invalid low water mark '%s' at line %d for group %s", lowWaterStr, lineNum, groupName) + continue + } + + // Calculate article count (high - low) + articleCount := highWater - lowWater + if articleCount < 0 { + articleCount = 0 + } + + // Apply min articles filter + if minArticles > 0 && articleCount < minArticles { + continue + } + + filteredGroups++ + + // Check if group exists in database + if _, exists := existingGroups[groupName]; !exists { + missingGroups++ + missingGroupsList = append(missingGroupsList, fmt.Sprintf("%s (articles: %d, high: %d, low: %d, status: %s)", + groupName, articleCount, highWater, lowWater, status)) + } + + // Log progress every 10000 lines + if lineNum%10000 == 0 { + log.Printf("[WEB]: Processed %d lines, found %d missing groups so far...", lineNum, missingGroups) + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading active file: %w", err) + } + + // Print results + fmt.Printf("=== COMPARISON RESULTS ===\n") + fmt.Printf("Total groups in active file: %d\n", totalGroupsInFile) + fmt.Printf("Groups after min articles filter (%d): %d\n", minArticles, filteredGroups) + fmt.Printf("Groups missing from database: %d\n\n", missingGroups) + + if missingGroups > 0 { + fmt.Printf("=== MISSING GROUPS ===\n") + for i, group := range missingGroupsList { + fmt.Printf("%d. %s\n", i+1, group) + } + fmt.Printf("\n") + } else { + fmt.Printf("✓ All groups from active file (meeting criteria) are present in database!\n\n") + } + + log.Printf("[WEB]: Active file comparison completed - %d total, %d filtered, %d missing", totalGroupsInFile, filteredGroups, missingGroups) + return nil +} + +// startHierarchyUpdater runs a background job every N minutes to update +// hierarchy last_updated fields based on their child newsgroups +func startHierarchyUpdater(db *database.Database) { + // Run immediately on startup + if err := db.UpdateHierarchiesLastUpdated(); err != nil { + log.Printf("[WEB]: Initial hierarchy update failed: %v", err) + } else { + // Update the hierarchy cache with new last_updated values + if db.HierarchyCache != nil { + if err := db.HierarchyCache.UpdateHierarchyLastUpdated(db); err != nil { + log.Printf("[WEB]: Initial hierarchy cache update failed: %v", err) + } else { + log.Printf("[WEB]: Initial hierarchy cache updated successfully") + } + } + } + log.Printf("[WEB]: Hierarchy updater started, will sync hierarchy last_updated every 30 minutes") + + for { + time.Sleep(10 * time.Minute) + if err := db.UpdateHierarchiesLastUpdated(); err != nil { + log.Printf("[WEB]: Hierarchy update failed: %v", err) + } else { + // Update the hierarchy cache with new last_updated values + if db.HierarchyCache != nil { + if err := db.HierarchyCache.UpdateHierarchyLastUpdated(db); err != nil { + log.Printf("[WEB]: Hierarchy cache update failed: %v", err) + } else { + log.Printf("[WEB]: Hierarchy cache updated successfully") + } + } + } + } +} + +// monitorUpdateFile checks for the existence of an .update file every 30 seconds +// and signals for shutdown when found, then removes the file +func monitorUpdateFile(shutdownChan chan<- bool) { + updateFilePath := ".update" + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + + log.Printf("[WEB]: Update file monitor started, checking for '%s' every 60 seconds", updateFilePath) + + for range ticker.C { + + // Check if .update file exists + if _, err := os.Stat(updateFilePath); err == nil { + log.Printf("[WEB]: Update file '%s' detected, triggering graceful shutdown", updateFilePath) + + // Rename the update file + if err := os.Rename(updateFilePath, updateFilePath+".todo"); err != nil { + log.Printf("[WEB]: Warning: Failed to rename update file '%s': %v", updateFilePath, err) + continue + } else { + log.Printf("[WEB]: Update file '%s' renamed successfully", updateFilePath) + } + + // Signal shutdown + select { + case shutdownChan <- true: + log.Printf("[WEB]: Shutdown signal sent via update file monitor") + default: + log.Printf("[WEB]: Shutdown channel already signaled") + } + return + } + // File doesn't exist, continue monitoring + } +} + +func ConnectPools(db *database.Database) []*nntp.Pool { + log.Printf("[WEB]: Fetching providers from database...") + providers, err := db.GetProviders() + if err != nil { + log.Fatalf("[WEB]: Failed to fetch providers: %v", err) + } + log.Printf("[WEB]: Found %d providers in database", len(providers)) + pools := make([]*nntp.Pool, 0, len(providers)) + log.Printf("[WEB]: Create provider connection pools...") + enabledProviders := 0 + for _, p := range providers { + if !p.Enabled || p.Host == "" || p.Port <= 0 { + log.Printf("[WEB]: Skipping disabled/invalid provider: %s (ID: %d, Enabled: %v, Host: '%s', Port: %d)", + p.Name, p.ID, p.Enabled, p.Host, p.Port) + continue // Skip disabled providers + } + enabledProviders++ + // Convert models.Provider to config.Provider for the BackendConfig + configProvider := &config.Provider{ + Grp: p.Grp, + Name: p.Name, + Host: p.Host, + Port: p.Port, + SSL: p.SSL, + Username: p.Username, + Password: p.Password, + MaxConns: p.MaxConns, + Enabled: p.Enabled, + Priority: p.Priority, + MaxArtSize: p.MaxArtSize, + // Copy proxy configuration from database provider + ProxyEnabled: p.ProxyEnabled, + ProxyType: p.ProxyType, + ProxyHost: p.ProxyHost, + ProxyPort: p.ProxyPort, + ProxyUsername: p.ProxyUsername, + ProxyPassword: p.ProxyPassword, + } + + backendConfig := &nntp.BackendConfig{ + Host: p.Host, + Port: p.Port, + SSL: p.SSL, + Username: p.Username, + Password: p.Password, + //ConnectTimeout: 9 * time.Second, + //ReadTimeout: 60 * time.Second, + //WriteTimeout: 60 * time.Second, + MaxConns: p.MaxConns, + Provider: configProvider, // Set the Provider field + // Copy proxy configuration from database provider + ProxyEnabled: p.ProxyEnabled, + ProxyType: p.ProxyType, + ProxyHost: p.ProxyHost, + ProxyPort: p.ProxyPort, + ProxyUsername: p.ProxyUsername, + ProxyPassword: p.ProxyPassword, + } + + pool := nntp.NewPool(backendConfig) + pool.StartCleanupWorker(5 * time.Second) + pools = append(pools, pool) + log.Printf("[WEB]: Using only first enabled provider '%s' (TODO: support multiple providers)", p.Name) + break // For now, we only use the first enabled provider!!! TODO + } + log.Printf("[WEB]: %d providers, %d enabled, using %d pools", len(providers), enabledProviders, len(pools)) + return pools +} + +func NewFetchProcessor(db *database.Database) *processor.Processor { + var pool *nntp.Pool + pools := ConnectPools(db) // Get NNTP pools + if len(pools) == 0 { + log.Printf("[WEB]: ERROR: No enabled providers found! Cannot proceed with article fetching") + pool = nil + } else { + pool = pools[0] + } + log.Printf("[WEB]: Creating processor instance with useShortHashLen=%d...", useShortHashLen) + proc := processor.NewProcessor(db, pool, useShortHashLen) // Create a new processor instance + log.Printf("[WEB]: Processor created successfully") + return proc +} + +func FetchRoutine(db *database.Database, proc *processor.Processor, useShortHashLen int, boot bool, isleep int64, DLParChan chan struct{}, progressDB *database.ProgressDB) { + /* DISABLED + if isleep < 15 { + isleep = 15 // min 15 sec sleep! + } + startTime := time.Now() + log.Printf("[WEB]: FetchRoutine STARTED (boot=%v, useShortHashLen=%d) at %v", boot, useShortHashLen, startTime) + + webmutex.Lock() + log.Printf("[WEB]: Acquired webmutex lock") + defer func() { + webmutex.Unlock() + log.Printf("[WEB]: Released webmutex lock") + }() + + if boot { + log.Printf("[WEB]: Boot mode detected - waiting %v before starting...", isleep) + select { + case <-db.StopChan: + log.Printf("[WEB]: Shutdown detected during boot wait, exiting FetchRoutine") + return + case <-time.After(time.Duration(isleep) * time.Second): + log.Printf("[WEB]: Boot wait completed, starting article fetching") + } + } + log.Printf("[WEB]: Begin article fetching process") + + defer func(isleep int64, progressDB *database.ProgressDB) { + duration := time.Since(startTime) + log.Printf("[WEB]: FetchRoutine COMPLETED after %v", duration) + if isleep > 30 { + proc.Pool.ClosePool() // Close the NNTP pool if sleep is more than 30 seconds + } + // Check if shutdown was requested before scheduling restart + select { + case <-db.StopChan: + log.Printf("[WEB]: Shutdown detected, not restarting FetchRoutine") + return + default: + // pass + } + // Sleep with shutdown check + wait: + for { + select { + case <-db.StopChan: + log.Printf("[WEB]: Shutdown detected during sleep, not restarting FetchRoutine") + return + case <-time.After(time.Duration(isleep) * time.Second): + break wait + } + } + pools := ConnectPools(db) // Reconnect pools after sleep + if len(pools) == 0 { + log.Printf("[WEB]: ERROR: No enabled providers found after sleep! Cannot proceed with article fetching") + } else { + proc.Pool = pools[0] // Use the first pool + } + log.Printf("[WEB]: Sleep completed, starting new FetchRoutine goroutine") + go FetchRoutine(db, proc, useShortHashLen, false, isleep, DLParChan, progressDB) + log.Printf("[WEB]: New FetchRoutine goroutine launched") + }(isleep, progressDB) + + log.Printf("[WEB]: Fetching newsgroups from database...") + groups, err := db.MainDBGetAllNewsgroups() + if err != nil { + log.Printf("[WEB]: FATAL: Could not fetch newsgroups: %v", err) + log.Printf("[WEB]: Warning: Could not fetch newsgroups: %v", err) + return + } + + log.Printf("[WEB]: Found %d newsgroups in database", len(groups)) + + // Check for data integrity issues + emptyNameCount := 0 + for _, group := range groups { + if group.Name == "" { + emptyNameCount++ + log.Printf("[WEB]: WARNING: Found newsgroup with empty name (ID: %d)", group.ID) + } + } + if emptyNameCount > 0 { + log.Printf("[WEB]: WARNING: Found %d newsgroups with empty names - this indicates a data integrity issue", emptyNameCount) + } + + // Configure bridges if enabled + if enableFediverse || enableMatrix { + bridgeConfig := &processor.BridgeConfig{ + FediverseEnabled: enableFediverse, + FediverseDomain: fediverseDomain, + FediverseBaseURL: fediverseBaseURL, + MatrixEnabled: enableMatrix, + MatrixHomeserver: matrixHomeserver, + MatrixAccessToken: matrixAccessToken, + MatrixUserID: matrixUserID, + } + proc.EnableBridges(bridgeConfig) + log.Printf("[WEB]: Bridge configuration applied") + } else { + log.Printf("[WEB]: No bridges enabled (use --enable-fediverse or --enable-matrix to enable)") + } + + log.Printf("[WEB]: Starting to process %d newsgroups...", len(groups)) + processedCount := 0 + upToDateCount := 0 + errorCount := 0 + + for i, group := range groups { + // Check for shutdown before processing each group + select { + case <-db.StopChan: + log.Printf("[WEB]: Shutdown detected, stopping newsgroup processing at group %d/%d", i+1, len(groups)) + return + default: + // Continue processing + } + + // Skip groups with empty names + if group.Name == "" { + log.Printf("[WEB]: [%d/%d] Skipping group with empty name (ID: %d)", i+1, len(groups), group.ID) + errorCount++ + continue + } + + log.Printf("[WEB]: [%d/%d] Processing group: %s (ID: %d)", i+1, len(groups), group.Name, group.ID) + + // Register newsgroup with bridges if enabled + if proc.BridgeManager != nil { + if err := proc.BridgeManager.RegisterNewsgroup(group); err != nil { + log.Printf("web.main.go: [%d/%d] Warning: Failed to register group %s with bridges: %v", i+1, len(groups), group.Name, err) + } + } + + // Check if the group is in sections DB + err := proc.DownloadArticles(group.Name, ignoreInitialTinyGroups, DLParChan, progressDB, 0 ,0) + if err != nil { + if err.Error() == "up2date" { + log.Printf("[WEB]: [%d/%d] Group %s is up to date, skipping", i+1, len(groups), group.Name) + upToDateCount++ + } else { + log.Printf("[WEB]: [%d/%d] ERROR processing group %s: %v", i+1, len(groups), group.Name, err) + errorCount++ + + // If we're getting too many consecutive errors, it might be an auth issue + if errorCount > 10 && (processedCount+upToDateCount) == 0 { + log.Printf("[WEB]: WARNING: Too many consecutive errors (%d), this might indicate authentication or connection issues", errorCount) + } + } + } else { + log.Printf("[WEB]: [%d/%d] Successfully processed group %s", i+1, len(groups), group.Name) + processedCount++ + } + log.Printf("[WEB]: Progress: processed=%d, up-to-date=%d, errors=%d, remaining=%d", + processedCount, upToDateCount, errorCount, len(groups)-(i+1)) + } + log.Printf("[WEB]: Finished processing all %d groups", len(groups)) + log.Printf("[WEB]: FINAL SUMMARY - Total groups: %d, Successfully processed: %d, Up-to-date: %d, Errors: %d", + len(groups), processedCount, upToDateCount, errorCount) + */ +} diff --git a/createChecksums.sh b/createChecksums.sh index e6efdd05..9e4c92e0 100755 --- a/createChecksums.sh +++ b/createChecksums.sh @@ -33,13 +33,6 @@ cd "$BUILD_DIR" sha256sum * > "../${CHECKSUMS_FILE}.archive" cd .. -# Display the checksums file -echo "" -echo "Checksums generated successfully:" -echo "=================================" -cat "$CHECKSUMS_FILE" - -echo "" echo "Checksums file created: $CHECKSUMS_FILE" echo "Archive checksums file created: ${CHECKSUMS_FILE}.archive (for inclusion in release)" -echo "Number of executables: $(wc -l < $CHECKSUMS_FILE)" \ No newline at end of file +echo "Number of executables: $(wc -l < $CHECKSUMS_FILE)" diff --git a/getUpdate.sh b/getUpdate.sh.template similarity index 50% rename from getUpdate.sh rename to getUpdate.sh.template index be004709..164241a7 100755 --- a/getUpdate.sh +++ b/getUpdate.sh.template @@ -4,7 +4,8 @@ test ! -e "$FILE" && echo "update file not found" && exit 1 mkdir -p tmp rm -f tmp/* test -e update.tar.gz && rm -v update.tar.gz -wget https://pugleaf.net/cdn/go-pugleaf/update.tar.gz -O tmp/update.tar.gz && cd tmp && \ +mv .update.todo .update.todo.downloading +wget XXXURLXXX -O tmp/update.tar.gz && cd tmp && \ sha256sum update.tar.gz && du -b update.tar.gz && \ - sha256sum -c ../$FILE && tar xfvz update.tar.gz && mv * ../new/ -rm -v ../"$FILE" + sha256sum -c ../.update.todo.downloading && tar xfvz update.tar.gz && mv * ../new/ +rm -v ../.update.todo.downloading diff --git a/go.mod b/go.mod index 3ccefa18..13c17787 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-while/go-cpu-mem-profiler v0.0.0-20240612221627-856954a5fc83 github.com/mattn/go-sqlite3 v1.14.32 golang.org/x/crypto v0.41.0 + golang.org/x/net v0.43.0 golang.org/x/term v0.34.0 golang.org/x/text v0.28.0 ) @@ -35,7 +36,6 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/net v0.43.0 // indirect golang.org/x/sys v0.35.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/internal/cache/newsgroup_cache.go b/internal/cache/newsgroup_cache.go index d1abfaa7..a1955173 100644 --- a/internal/cache/newsgroup_cache.go +++ b/internal/cache/newsgroup_cache.go @@ -9,7 +9,7 @@ import ( // Newsgroup represents a newsgroup for caching (avoiding import cycle) type Newsgroup struct { - ID int `json:"id"` + ID int64 `json:"id"` Name string `json:"name"` Active bool `json:"active"` Description string `json:"description"` diff --git a/internal/common/headers.go b/internal/common/headers.go new file mode 100644 index 00000000..c18821d6 --- /dev/null +++ b/internal/common/headers.go @@ -0,0 +1,179 @@ +// Package common provides shared utilities for go-pugleaf +package common + +import ( + "fmt" + "log" + "strings" + "time" + "unicode" + + "github.com/go-while/go-pugleaf/internal/models" +) + +// IgnoreHeadersMap is a map version of IgnoreHeaders for fast lookup +var IgnoreHeadersMap = map[string]bool{ + "message-id": true, + "subject": true, + "from": true, + "date": true, + "references": true, + "path": true, + "xref": true, +} + +var formats = []string{ + time.RFC1123Z, // "Mon, 02 Jan 2006 15:04:05 -0700" + time.RFC1123, // "Mon, 02 Jan 2006 15:04:05 MST" + time.RFC822, // "02 Jan 06 15:04 MST" + time.RFC822Z, // "02 Jan 06 15:04 -0700" + "Mon, _2 Jan 2006 15:04:05 MST", // Single digit day + "Mon, _2 Jan 2006 15:04:05 -0700", // Single digit day with timezone +} + +// isRFCdate checks if a date string is RFC compliant for Usenet +func isRFCdate(dateStr string) bool { + // Try to parse with common RFC formats used in Usenet + + for _, format := range formats { + if _, err := time.Parse(format, dateStr); err == nil { + return true + } + } + return false +} + +const pathHeader_inv1 string = "Path: pugleaf.invalid!.TX!not-for-mail" +const pathHeader_inv2 string = "X-Path: pugleaf.invalid!.TX!not-for-mail" + +// ReconstructHeaders reconstructs the header lines from an article for transmission +func ReconstructHeaders(article *models.Article, withPath bool, nntphostname *string) ([]string, error) { + var headers []string + + // Add basic headers that we know about + if article.MessageID == "" { + return nil, fmt.Errorf("article missing Message-ID") + } + if article.Subject == "" { + return nil, fmt.Errorf("article missing Subject") + } + if article.FromHeader == "" { + return nil, fmt.Errorf("article missing From header") + } + + // Check if DateString is RFC Usenet compliant, use DateSent if not + var dateHeader string + if article.DateString != "" { + // Check if DateString is RFC-compliant by trying to parse it + if isRFCdate(article.DateString) { + dateHeader = article.DateString + } else { + // DateString is not RFC compliant, use DateSent instead + if !article.DateSent.IsZero() { + dateHeader = article.DateSent.UTC().Format(time.RFC1123Z) + if VerboseHeaders { + log.Printf("Using dateHeader '%s' instead of DateString '%s' for article %s", dateHeader, article.DateString, article.MessageID) + } + } else { + return nil, fmt.Errorf("article has non-compliant DateString and zero DateSent msgId='%s'", article.MessageID) + } + } + } else { + // No DateString, try DateSent + if !article.DateSent.IsZero() { + dateHeader = article.DateSent.UTC().Format(time.RFC1123) + } else { + return nil, fmt.Errorf("article missing Date header (both DateString and DateSent are empty) msgId='%s'", article.MessageID) + } + } + headers = append(headers, "Message-ID: "+article.MessageID) + headers = append(headers, "Subject: "+article.Subject) + headers = append(headers, "From: "+article.FromHeader) + headers = append(headers, "Date: "+dateHeader) + if article.References != "" { + headers = append(headers, "References: "+article.References) + } + switch withPath { + case true: + if article.Path != "" { + if nntphostname != nil && *nntphostname != "" { + headers = append(headers, "Path: "+*nntphostname+"!.TX!"+article.Path) + } else { + headers = append(headers, "Path: "+article.Path) + } + } else { + headers = append(headers, pathHeader_inv1) + } + case false: + if article.Path != "" { + if nntphostname != nil && *nntphostname != "" { + headers = append(headers, "X-Path: "+*nntphostname+"!.TX!"+article.Path) + } else { + headers = append(headers, "X-Path: "+article.Path) + } + } else { + headers = append(headers, pathHeader_inv2) + } + } + moreHeaders := strings.Split(article.HeadersJSON, "\n") + ignoreLine := false + isSpacedLine := false + ignoredLines := 0 + headersMap := make(map[string]bool) + + for i, headerLine := range moreHeaders { + if len(headerLine) == 0 { + log.Printf("Empty headerline=%d in msgId='%s' (continue)", i, article.MessageID) + continue + } + isSpacedLine = strings.HasPrefix(headerLine, " ") || strings.HasPrefix(headerLine, "\t") + if isSpacedLine && ignoreLine { + ignoredLines++ + continue + } else { + ignoreLine = false + } + if !isSpacedLine { + if len(headerLine) < 4 { // "X: A" + log.Printf("Short header: '%s' line=%d in msgId='%s' (continue)", headerLine, i, article.MessageID) + ignoreLine = true + ignoredLines++ + continue + } + // check if first char is lowercase + if unicode.IsLower(rune(headerLine[0])) { + headerLine = strings.ToUpper(string(headerLine[0])) + headerLine[1:] + if VerboseHeaders { + log.Printf("Lowercase header: '%s' line=%d in msgId='%s' (rewrote)", headerLine, i, article.MessageID) + } + } + header := strings.SplitN(headerLine, ":", 2)[0] + if len(header) == 0 { + log.Printf("Invalid header: '%s' line=%d in msgId='%s' (continue)", headerLine, i, article.MessageID) + ignoreLine = true + ignoredLines++ + continue + } + if IgnoreHeadersMap[strings.ToLower(header)] { + ignoreLine = true + continue + } + + if !strings.HasPrefix(header, "X-") { + if headersMap[strings.ToLower(header)] { + log.Printf("Duplicate header: '%s' line=%d in msgId='%s' (continue)", headerLine, i, article.MessageID) + ignoreLine = true + continue + } + headersMap[strings.ToLower(header)] = true + } + } + headers = append(headers, headerLine) + } + if VerboseHeaders && ignoredLines > 0 { + log.Printf("Reconstructed %d header lines, ignored %d: msgId='%s'", len(headers), ignoredLines, article.MessageID) + } + return headers, nil +} + +var VerboseHeaders = false diff --git a/internal/config/config.go b/internal/config/config.go index cf4ff42c..b7627a35 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,6 +62,14 @@ type Provider struct { Enabled bool `json:"enabled"` Priority int `json:"priority"` // Lower numbers = higher priority MaxArtSize int `json:"max_article_size"` // Maximum article size in bytes + Posting bool `json:"posting"` // Whether posting is enabled for this provider + // Proxy configuration fields + ProxyEnabled bool `json:"proxy_enabled"` // Whether to use proxy for this provider + ProxyType string `json:"proxy_type"` // Proxy type: socks4, socks5 + ProxyHost string `json:"proxy_host"` // Proxy server hostname/IP + ProxyPort int `json:"proxy_port"` // Proxy server port + ProxyUsername string `json:"proxy_username"` // Proxy authentication username + ProxyPassword string `json:"proxy_password"` // Proxy authentication password } // ServerConfig holds Web and NNTP server configuration diff --git a/internal/database/db_batch.go b/internal/database/db_batch.go index 2613d806..8d25790e 100644 --- a/internal/database/db_batch.go +++ b/internal/database/db_batch.go @@ -18,8 +18,8 @@ var BatchInterval = 3 * time.Second var MaxBatchSize int = 100 // don't process more than N groups in parallel: better have some cpu & mem when importing hard! -var MaxBatchThreads = 16 - +var MaxBatchThreads = 16 // -max-batch-threads N +var MaxQueued = 16384 // -max-queue N var InitialBatchChannelSize = MaxBatchSize // @AI: DO NOT CHANGE THIS!!!! per group cache channel size. should be less or equal to MaxBatch in processor aka MaxReadLinesXover in nntp-client-commands // Cache for placeholder strings to avoid rebuilding them repeatedly @@ -32,16 +32,24 @@ func getPlaceholders(count int) string { if count <= 0 { return "" } - if v, ok := placeholderCache.Load(count); ok { - return v.(string) + + if count == MaxBatchSize { + if v, ok := placeholderCache.Load(count); ok { + return v.(string) + } } + var s string + if count == 1 { s = "?" } else { s = strings.Repeat("?, ", count-1) + "?" } - placeholderCache.Store(count, s) + + if count == MaxBatchSize { + placeholderCache.Store(count, s) + } return s } @@ -75,13 +83,15 @@ type ThreadingProcessor interface { // Add method for finding thread roots - matches proc_MsgIDtmpCache.go signature (updated to use pointer) FindThreadRootInCache(groupName *string, refs []string) *MsgIdTmpCacheItem CheckNoMoreWorkInHistory() bool + // Add method for force closing group databases + ForceCloseGroupDBs(groupsDB *GroupDBs) error } // SetProcessor sets the threading processor callback interface func (c *SQ3batch) SetProcessor(proc ThreadingProcessor) { c.proc = proc if MaxBatchSize > 1000 { - log.Printf("[BATCH] MaxBatchSize is set to %d, reduced to 1000 in db_batch", MaxBatchSize) + //log.Printf("[BATCH] MaxBatchSize is set to %d, reduced to 1000 in db_batch", MaxBatchSize) MaxBatchSize = 1000 } } @@ -98,6 +108,7 @@ type SQ3batch struct { TmpStringSlices chan []string // holds temporary string slices for valuesClauses, messageIDs etc TmpStringPtrSlices chan []*string // holds temporary string slices for valuesClauses, messageIDs etc TmpInterfaceSlices chan []interface{} // holds temporary interface slices for args in threading operations + queued int // number of queued articles for batch processing } type BatchTasks struct { @@ -113,12 +124,12 @@ type BatchTasks struct { func NewSQ3batch(db *Database) *SQ3batch { batch := &SQ3batch{ db: db, - TasksMap: make(map[string]*BatchTasks, 128), // Initialize TasksMap with a capacity for initial cap of 4096 Newsgroups - TmpTasksChans: make(chan chan *BatchTasks, 128), // Initialize TasksMap with a capacity for initial cap of 4096 Newsgroups - TmpArticleSlices: make(chan []*models.Article, 128), // Initialize TasksMap with a capacity for initial cap of 4096 Newsgroups - TmpStringSlices: make(chan []string, 128), // Initialize string slice pool for valuesClauses, messageIDs etc - TmpStringPtrSlices: make(chan []*string, 128), // Initialize string pointer slice pool for messageIDs etc - TmpInterfaceSlices: make(chan []interface{}, 128), // Initialize interface slice pool for args in threading operations + TasksMap: make(map[string]*BatchTasks, 128), // Initialize TasksMap + TmpTasksChans: make(chan chan *BatchTasks, 128), + TmpArticleSlices: make(chan []*models.Article, 128), + TmpStringSlices: make(chan []string, 128), + TmpStringPtrSlices: make(chan []*string, 128), + TmpInterfaceSlices: make(chan []interface{}, 128), } batch.orchestrator = NewBatchOrchestrator(batch) return batch @@ -312,7 +323,7 @@ func (c *SQ3batch) getOrCreateTmpTasksChan() (tasksChan chan *BatchTasks) { } // processAllPendingBatches processes all pending batches in the correct sequential order -func (c *SQ3batch) processAllPendingBatches(wgProcessAllBatches *sync.WaitGroup) { +func (c *SQ3batch) processAllPendingBatches(wgProcessAllBatches *sync.WaitGroup, limit int) { if !LockPending() { log.Printf("[BATCH] processAllPendingBatches: LockPending failed") return @@ -333,7 +344,7 @@ fill: continue } task.Mux.RUnlock() - if len(task.BATCHchan) > 0 { + if len(task.BATCHchan) > 0 && len(task.BATCHchan) <= limit { select { case tasksToProcess <- task: // Send the task to the channel default: @@ -505,7 +516,7 @@ func (c *SQ3batch) getOrCreateInterfaceSlice() []interface{} { return make([]interface{}, 0, MaxBatchSize*3) // up to 3x for reply count updates } -func (c *SQ3batch) processNewsgroupBatch(task *BatchTasks) { +func (sq *SQ3batch) processNewsgroupBatch(task *BatchTasks) { startTime := time.Now() task.Mux.Lock() task.Expires = time.Now().Add(120 * time.Second) // extend expiration time @@ -522,8 +533,8 @@ func (c *SQ3batch) processNewsgroupBatch(task *BatchTasks) { }(task, startTime) // Collect all batches for this newsgroup - batches := c.getOrCreateModelsArticleSlice() - defer c.returnModelsArticleSlice(batches) + batches := sq.getOrCreateModelsArticleSlice() + defer sq.returnModelsArticleSlice(batches) // Drain the channel drainChannel: @@ -541,28 +552,24 @@ drainChannel: } log.Printf("[BATCH] processNewsgroupBatch: ng: '%s' with %d articles (more queued: %d)", *task.Newsgroup, len(batches), len(task.BATCHchan)) - deferred := false - var groupDBs *GroupDBs - var err error -retry: - if groupDBs == nil { - // Get database connection for this newsgroup - groupDBs, err = c.db.GetGroupDBs(*task.Newsgroup) - if err != nil { - log.Printf("[BATCH] processNewsgroupBatch Failed to get database for group '%s': %v", *task.Newsgroup, err) - return - } - } - if !deferred { - defer groupDBs.Return(c.db) // Ensure proper cleanup - deferred = true + +retry1: + // Get database connection for this newsgroup + groupDBs, err := sq.db.GetGroupDBs(*task.Newsgroup) + if err != nil { + log.Printf("[BATCH] processNewsgroupBatch Failed to get database for group '%s': %v", *task.Newsgroup, err) + return } // PHASE 1: Insert complete articles (overview + article data unified) and set article numbers directly on batches - if err := c.batchInsertOverviews(*task.Newsgroup, batches, groupDBs); err != nil { - log.Printf("[BATCH] processNewsgroupBatch Failed to process batch for group '%s': %v", *task.Newsgroup, err) + if err := sq.batchInsertOverviews(*task.Newsgroup, batches, groupDBs, task.Newsgroup); err != nil { + if groupDBs != nil { + sq.proc.ForceCloseGroupDBs(groupDBs) + log.Printf("[BATCH] processNewsgroupBatch Failed1 to process batch for group '%s': %v groupDBs='%#v'", *task.Newsgroup, err, groupDBs) + groupDBs = nil + } time.Sleep(time.Second) - goto retry + goto retry1 } // Update all articles with their assigned numbers for subsequent processing @@ -582,12 +589,25 @@ retry: // PHASE 2: Process threading for all articles (reusing the same DB connection) //log.Printf("[BATCH] processNewsgroupBatch Starting threading phase for %d articles in group '%s'", len(batches), *task.Newsgroup) //start := time.Now() - err = c.batchProcessThreading(task.Newsgroup, batches, groupDBs) - //threadingDuration := time.Since(start) - if err != nil { - log.Printf("[BATCH] processNewsgroupBatch Failed to process threading for group '%s': %v", *task.Newsgroup, err) - return +retry2: + if groupDBs == nil { + groupDBs, err = sq.db.GetGroupDBs(*task.Newsgroup) + if err != nil { + log.Printf("[BATCH] processNewsgroupBatch Failed2 to get database for group '%s': %v", *task.Newsgroup, err) + return + } } + if err := sq.batchProcessThreading(task.Newsgroup, batches, groupDBs); err != nil { + time.Sleep(time.Second) + if groupDBs != nil { + groupDBs.Return(sq.db) + log.Printf("[BATCH] processNewsgroupBatch Failed2 to process threading for group '%s': %v groupDBs='%#v'", *task.Newsgroup, err, groupDBs) + groupDBs = nil + } + goto retry2 + } + defer groupDBs.Return(sq.db) + //threadingDuration := time.Since(start) //log.Printf("[BATCH] processNewsgroupBatch Completed threading phase for group '%s' in %v", *task.Newsgroup, threadingDuration) // PHASE 3: Handle history and processor cache updates @@ -598,7 +618,7 @@ retry: //log.Printf("[BATCH] processNewsgroupBatch Updating history/cache for article %d/%d in group '%s'", i+1, len(batches), *task.Newsgroup) // Read article number under read lock to avoid concurrent map access article.Mux.RLock() - c.proc.AddProcessedArticleToHistory(article.MsgIdItem, task.Newsgroup, article.ArticleNums[task.Newsgroup]) + sq.proc.AddProcessedArticleToHistory(article.MsgIdItem, task.Newsgroup, article.ArticleNums[task.Newsgroup]) article.Mux.RUnlock() article.Mux.Lock() if len(article.NewsgroupsPtr) > 0 { @@ -642,14 +662,14 @@ retry: //log.Printf("[BATCH] processNewsgroupBatch Completed history/cache updates for group '%s' in %v", *task.Newsgroup, historyDuration) // Update newsgroup statistics with retryable transaction to avoid race conditions // Safety check for nil database connection - if c.db == nil || c.db.mainDB == nil { + if sq.db == nil || sq.db.mainDB == nil { log.Printf("[BATCH] processNewsgroupBatch Main database connection is nil, cannot update newsgroup stats for '%s'", *task.Newsgroup) err = fmt.Errorf("processNewsgroupBatch main database connection is nil") } else { //LockQueryChan() //defer ReturnQueryChan() // Use retryable transaction to prevent race conditions between concurrent batches - err = retryableTransactionExec(c.db.mainDB, func(tx *sql.Tx) error { + err = retryableTransactionExec(sq.db.mainDB, func(tx *sql.Tx) error { // Use UPSERT to handle both new and existing newsgroups _, txErr := tx.Exec(query_processNewsgroupBatch, *task.Newsgroup, len(batches), maxArticleNum, time.Now().UTC().Format("2006-01-02 15:04:05")) @@ -663,26 +683,28 @@ retry: //log.Printf("[BATCH] processNewsgroupBatch Updated newsgroup '%s' stats: +%d articles, max_article=%d", *task.Newsgroup, increment, maxArticleNum) // Update hierarchy cache with new stats instead of invalidating - if c.db.HierarchyCache != nil { - c.db.HierarchyCache.UpdateNewsgroupStats(*task.Newsgroup, len(batches), maxArticleNum) + if sq.db.HierarchyCache != nil { + sq.db.HierarchyCache.UpdateNewsgroupStats(*task.Newsgroup, len(batches), maxArticleNum) } - } if err != nil { log.Printf("[BATCH] processNewsgroupBatch Failed to update newsgroup stats for '%s': %v", *task.Newsgroup, err) } log.Printf("[BATCH-END] newsgroup '%s' processed articles: %d (took %v)", *task.Newsgroup, len(batches), time.Since(startTime)) + sq.GMux.Lock() + sq.queued -= len(batches) + sq.GMux.Unlock() } // batchInsertOverviews - now sets ArticleNum directly on each batch's Article and reuses the GroupDBs connection -func (c *SQ3batch) batchInsertOverviews(newsgroup string, batches []*models.Article, groupDBs *GroupDBs) error { +func (c *SQ3batch) batchInsertOverviews(newsgroup string, batches []*models.Article, groupDBs *GroupDBs, taskNewsgroup *string) error { if len(batches) == 0 { return fmt.Errorf("no batches to process for group '%s'", newsgroup) } if len(batches) <= MaxBatchSize { // Small batch - process directly - if err := c.processOverviewBatch(groupDBs, batches); err != nil { + if err := c.processOverviewBatch(groupDBs, batches, taskNewsgroup); err != nil { log.Printf("[OVB-BATCH] Failed to process small batch for group '%s': %v", newsgroup, err) return fmt.Errorf("failed to process small batch for group '%s': %w", newsgroup, err) } @@ -696,7 +718,7 @@ func (c *SQ3batch) batchInsertOverviews(newsgroup string, batches []*models.Arti end = len(batches) } - if err := c.processOverviewBatch(groupDBs, batches[i:end]); err != nil { + if err := c.processOverviewBatch(groupDBs, batches[i:end], taskNewsgroup); err != nil { log.Printf("[OVB-BATCH] Failed to process chunk %d-%d for group '%s': %v", i, end, newsgroup, err) return fmt.Errorf("failed to process chunk %d-%d for group '%s': %w", i, end, newsgroup, err) } @@ -708,7 +730,7 @@ const query_processOverviewBatch = `INSERT OR IGNORE INTO articles (message_id, const query_processOverviewBatch2 = `SELECT message_id, article_num FROM articles WHERE message_id IN (` // processSingleUnifiedArticleBatch handles a single batch that's within SQLite limits -func (c *SQ3batch) processOverviewBatch(groupDBs *GroupDBs, batches []*models.Article) error { +func (c *SQ3batch) processOverviewBatch(groupDBs *GroupDBs, batches []*models.Article, taskNewsgroup *string) error { // Get timestamp once for the entire batch instead of per article importedAt := time.Now() @@ -729,13 +751,12 @@ func (c *SQ3batch) processOverviewBatch(groupDBs *GroupDBs, batches []*models.Ar // Execute the prepared statement for each batch item for _, article := range batches { // Format DateSent as UTC string to avoid timezone encoding issues - dateSentStr := article.DateSent.UTC().Format("2006-01-02 15:04:05") _, err := stmt.Exec( - article.MessageID, // message_id - article.Subject, // subject - article.FromHeader, // from_header - dateSentStr, // date_sent (formatted as UTC string) + article.MessageID, // message_id + article.Subject, // subject + article.FromHeader, // from_header + article.DateSent.UTC().Format("2006-01-02 15:04:05"), // date_sent (formatted as UTC string) article.DateString, // date_string article.References, // references article.Bytes, // bytes @@ -771,7 +792,6 @@ func (c *SQ3batch) processOverviewBatch(groupDBs *GroupDBs, batches []*models.Ar return fmt.Errorf("failed to execute batch select for group '%s': %w", groupDBs.Newsgroup, err) } defer rows.Close() - newsgroupPtr := c.GetNewsgroupPointer(groupDBs.Newsgroup) var messageID string var articleNum, idToArticleNum, timeSpent, spentms, loops int64 @@ -791,10 +811,10 @@ func (c *SQ3batch) processOverviewBatch(groupDBs *GroupDBs, batches []*models.Ar // Assign article numbers directly to batch Articles article.Mux.Lock() if article.MessageID == messageID { - if article.ArticleNums[newsgroupPtr] == 0 { - article.ArticleNums[newsgroupPtr] = articleNum + if article.ArticleNums[taskNewsgroup] == 0 { + article.ArticleNums[taskNewsgroup] = articleNum } else { - log.Printf("[OVB-BATCH] group '%s': Article with message_id %s already assigned article number %d, did not reassign from db: %d", groupDBs.Newsgroup, messageID, article.ArticleNums[newsgroupPtr], articleNum) + log.Printf("[OVB-BATCH] group '%s': Article with message_id %s already assigned article number %d, did not reassign from db: %d", groupDBs.Newsgroup, messageID, article.ArticleNums[taskNewsgroup], articleNum) } article.Mux.Unlock() timeSpent += time.Since(startN).Microseconds() @@ -813,20 +833,20 @@ func (c *SQ3batch) processOverviewBatch(groupDBs *GroupDBs, batches []*models.Ar } // batchProcessThreading processes all threading operations using existing GroupDBs connection -func (c *SQ3batch) batchProcessThreading(groupName *string, batches []*models.Article, groupDBs *GroupDBs) error { +func (c *SQ3batch) batchProcessThreading(taskNewsgroup *string, batches []*models.Article, groupDBs *GroupDBs) error { if len(batches) == 0 { return nil } - //log.Printf("[THR-BATCH] group '%s': %d articles to process", *groupName, len(batches)) + //log.Printf("[THR-BATCH] group '%s': %d articles to process", *taskNewsgroup, len(batches)) roots, replies := 0, 0 - newsgroupPtr := c.GetNewsgroupPointer(groupDBs.Newsgroup) for _, article := range batches { if article == nil { continue } article.Mux.RLock() - if article.ArticleNums[newsgroupPtr] <= 0 { - log.Printf("[THR-BATCH] ERROR batchProcessThreading NewsgroupsPtr in %s, skipping msgId='%s'", *newsgroupPtr, article.MessageID) + if article.ArticleNums[taskNewsgroup] <= 0 { + // when reaching here, we got a duplicate in remote servers overview and we already processed the article earlier... + log.Printf("[THR-BATCH] INFO: Did not insert into '%s' skipping msgId='%s'", *taskNewsgroup, article.MessageID) article.Mux.RUnlock() continue } @@ -840,16 +860,16 @@ func (c *SQ3batch) batchProcessThreading(groupName *string, batches []*models.Ar // Process thread roots first (they need to exist before replies can reference them) if roots > 0 { - if err := c.batchProcessThreadRoots(groupDBs, batches); err != nil { - log.Printf("[THR-BATCH] group '%s': Failed to batch process thread roots: %v", *groupName, err) + if err := c.batchProcessThreadRoots(groupDBs, batches, taskNewsgroup); err != nil { + log.Printf("[THR-BATCH] group '%s': Failed to batch process thread roots: %v", *taskNewsgroup, err) // Continue processing - don't fail the whole batch } } // Process replies if replies > 0 { - if err := c.batchProcessReplies(groupDBs, batches); err != nil { - log.Printf("[THR-BATCH] group '%s': Failed to batch process replies: %v", *groupName, err) + if err := c.batchProcessReplies(groupDBs, batches, taskNewsgroup); err != nil { + log.Printf("[THR-BATCH] group '%s': Failed to batch process replies: %v", *taskNewsgroup, err) // Continue processing - don't fail the whole batch } } @@ -860,7 +880,7 @@ func (c *SQ3batch) batchProcessThreading(groupName *string, batches []*models.Ar const query_batchProcessThreadRoots = "INSERT INTO threads (root_article, parent_article, child_article, depth, thread_order) VALUES (?, ?, ?, 0, 0)" // batchProcessThreadRoots processes thread root articles in TRUE batch -func (c *SQ3batch) batchProcessThreadRoots(groupDBs *GroupDBs, rootBatches []*models.Article) error { +func (c *SQ3batch) batchProcessThreadRoots(groupDBs *GroupDBs, rootBatches []*models.Article, taskNewsgroup *string) error { if len(rootBatches) == 0 { return nil } @@ -868,19 +888,18 @@ func (c *SQ3batch) batchProcessThreadRoots(groupDBs *GroupDBs, rootBatches []*mo // Use a transaction with prepared statement for cleaner, more efficient execution tx, err := groupDBs.DB.Begin() if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) + return fmt.Errorf("failed to begin transaction in batchProcessThreadRoots: %w", err) } defer tx.Rollback() // Prepare the INSERT statement once stmt, err := tx.Prepare(query_batchProcessThreadRoots) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return fmt.Errorf("failed to prepare statement in batchProcessThreadRoots: %w", err) } defer stmt.Close() // Single loop: execute statement for each valid root article - newsgroupPtr := c.GetNewsgroupPointer(groupDBs.Newsgroup) processedCount := 0 var threadCacheEntries []struct { articleNum int64 @@ -893,18 +912,18 @@ func (c *SQ3batch) batchProcessThreadRoots(groupDBs *GroupDBs, rootBatches []*mo } article.Mux.RLock() - if article.ArticleNums[newsgroupPtr] <= 0 || !article.IsThrRoot { + if article.ArticleNums[taskNewsgroup] <= 0 || !article.IsThrRoot { article.Mux.RUnlock() continue } - articleNum := article.ArticleNums[newsgroupPtr] + articleNum := article.ArticleNums[taskNewsgroup] // Execute the prepared statement directly - for thread roots, parent_article is NULL _, err := stmt.Exec(articleNum, nil, articleNum) // root_article, parent_article, child_article if err != nil { article.Mux.RUnlock() - return fmt.Errorf("failed to execute thread insert for article %d: %w", articleNum, err) + return fmt.Errorf("failed to execute thread insert in batchProcessThreadRoots for article %d: %w", articleNum, err) } // Collect data for post-processing OUTSIDE the transaction @@ -923,7 +942,7 @@ func (c *SQ3batch) batchProcessThreadRoots(groupDBs *GroupDBs, rootBatches []*mo // Commit the transaction BEFORE doing thread cache operations if err := tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) + return fmt.Errorf("failed to commit transaction in batchProcessThreadRoots: %w", err) } // Do post-processing AFTER transaction is committed to avoid SQLite lock conflicts @@ -940,19 +959,10 @@ func (c *SQ3batch) batchProcessThreadRoots(groupDBs *GroupDBs, rootBatches []*mo } // batchProcessReplies processes reply articles in TRUE batch -func (c *SQ3batch) batchProcessReplies(groupDBs *GroupDBs, replyBatches []*models.Article) error { +func (c *SQ3batch) batchProcessReplies(groupDBs *GroupDBs, replyBatches []*models.Article, taskNewsgroup *string) error { if len(replyBatches) == 0 { return nil } - /* - // Get pooled collections to avoid repeated memory allocations - parentMsgIDsPtr := parentMessageIDsPool.Get().(*map[string]int) - parentMessageIDs := *parentMsgIDsPtr - // Clear the map before use - for k := range parentMessageIDs { - delete(parentMessageIDs, k) - } - */ parentMessageIDs := make(map[*string]int, MaxBatchSize) // Pre-allocate map with expected size defer func() { for k := range parentMessageIDs { @@ -966,17 +976,6 @@ func (c *SQ3batch) batchProcessReplies(groupDBs *GroupDBs, replyBatches []*model childDate time.Time }, 0, len(replyBatches)) // Pre-allocate slice with capacity - /* - defer func() { - // Clear and return parentMessageIDs to pool - for k := range parentMessageIDs { - delete(parentMessageIDs, k) - } - parentMessageIDsPool.Put(parentMsgIDsPtr) - }() - */ - - newsgroupPtr := c.GetNewsgroupPointer(groupDBs.Newsgroup) // Process each reply to gather data preAllocThreadRoots := 0 for _, article := range replyBatches { @@ -985,7 +984,7 @@ func (c *SQ3batch) batchProcessReplies(groupDBs *GroupDBs, replyBatches []*model } article.Mux.RLock() - if article.ArticleNums[newsgroupPtr] <= 0 || !article.IsReply { + if article.ArticleNums[taskNewsgroup] <= 0 || !article.IsReply { article.Mux.RUnlock() continue } @@ -1002,7 +1001,7 @@ func (c *SQ3batch) batchProcessReplies(groupDBs *GroupDBs, replyBatches []*model articleNum int64 threadRoot int64 childDate time.Time - }{article.ArticleNums[newsgroupPtr], threadRoot, article.DateSent}) + }{article.ArticleNums[taskNewsgroup], threadRoot, article.DateSent}) if threadRoot > 0 { preAllocThreadRoots++ } @@ -1016,23 +1015,25 @@ func (c *SQ3batch) batchProcessReplies(groupDBs *GroupDBs, replyBatches []*model } } - threadUpdates := make(map[int64][]threadCacheUpdateData, preAllocThreadRoots) - log.Printf("[P-BATCH] group '%s': Pre-allocated thread updates map with capacity %d", groupDBs.Newsgroup, preAllocThreadRoots) - for _, data := range replyData { - if data.threadRoot > 0 { - threadUpdates[data.threadRoot] = append(threadUpdates[data.threadRoot], threadCacheUpdateData{ - childArticleNum: data.articleNum, - childDate: data.childDate, - }) + if preAllocThreadRoots > 0 { + threadUpdates := make(map[int64][]threadCacheUpdateData, preAllocThreadRoots) + //log.Printf("[P-BATCH] group '%s': Pre-allocated thread updates map with capacity %d", groupDBs.Newsgroup, preAllocThreadRoots) + for _, data := range replyData { + if data.threadRoot > 0 { + threadUpdates[data.threadRoot] = append(threadUpdates[data.threadRoot], threadCacheUpdateData{ + childArticleNum: data.articleNum, + childDate: data.childDate, + }) + } } - } - // Execute ALL thread cache updates in a single transaction - if len(threadUpdates) > 0 { - if err := c.batchUpdateThreadCache(groupDBs, threadUpdates); err != nil { - log.Printf("[P-BATCH] group '%s': Failed to batch update thread cache: %v", groupDBs.Newsgroup, err) + // Execute ALL thread cache updates in a single transaction + if len(threadUpdates) > 0 { + if err := c.batchUpdateThreadCache(groupDBs, threadUpdates); err != nil { + log.Printf("[P-BATCH] group '%s': Failed to batch update thread cache: %v", groupDBs.Newsgroup, err) + } + log.Printf("[P-BATCH] group '%s': Updated thread cache for %d thread roots", groupDBs.Newsgroup, len(threadUpdates)) } - log.Printf("[P-BATCH] group '%s': Updated thread cache for %d thread roots", groupDBs.Newsgroup, len(threadUpdates)) } return nil @@ -1107,7 +1108,7 @@ func (c *SQ3batch) findThreadRoot(groupDBs *GroupDBs, refs []string) (int64, err } // batchUpdateThreadCache performs TRUE batch update of thread cache entries in a single transaction with retry logic -func (c *SQ3batch) batchUpdateThreadCache(groupDBs *GroupDBs, threadUpdates map[int64][]threadCacheUpdateData) error { +func (sq *SQ3batch) batchUpdateThreadCache(groupDBs *GroupDBs, threadUpdates map[int64][]threadCacheUpdateData) error { if len(threadUpdates) == 0 { return nil } @@ -1148,10 +1149,10 @@ func (c *SQ3batch) batchUpdateThreadCache(groupDBs *GroupDBs, threadUpdates map[ err := retryableStmtQueryRowScan(selectStmt, []interface{}{threadRoot}, ¤tChildren, ¤tCount) if err != nil { // Thread cache entry doesn't exist, initialize it with the first update - firstUpdate := updates[0] + //firstUpdate := updates[0] // Format dates as UTC strings to avoid timezone encoding issues - firstUpdateDateUTC := firstUpdate.childDate.UTC().Format("2006-01-02 15:04:05") - _, err = retryableStmtExec(initStmt, threadRoot, firstUpdateDateUTC, firstUpdate.childArticleNum, firstUpdateDateUTC) + firstUpdateDateUTC := updates[0].childDate.UTC().Format("2006-01-02 15:04:05") + _, err = retryableStmtExec(initStmt, threadRoot, firstUpdateDateUTC, updates[0].childArticleNum, firstUpdateDateUTC) if err != nil { log.Printf("[BATCH-CACHE] Failed to initialize thread cache for root %d after retries: %v", threadRoot, err) return fmt.Errorf("failed to initialize thread cache for root %d: %w", threadRoot, err) @@ -1193,8 +1194,8 @@ func (c *SQ3batch) batchUpdateThreadCache(groupDBs *GroupDBs, threadUpdates map[ updatedCount++ // Update memory cache if available - if c.db.MemThreadCache != nil { - c.db.MemThreadCache.UpdateThreadMetadata(groupDBs.Newsgroup, threadRoot, newCount, lastActivity, newChildren) + if sq.db.MemThreadCache != nil { + sq.db.MemThreadCache.UpdateThreadMetadata(groupDBs.Newsgroup, threadRoot, newCount, lastActivity, newChildren) } } @@ -1243,12 +1244,8 @@ func (o *BatchOrchestrator) StartOrch() { for { time.Sleep(time.Second / 2) if o.batch.db.IsDBshutdown() { - if o.batch.proc == nil { - log.Printf("[ORCHESTRATOR1] o.batch.proc not set. shutting down.") - return - } log.Printf("[ORCHESTRATOR1] Database shutdown detected ShutDownCounter=%d", ShutDownCounter) - o.batch.processAllPendingBatches(&wgProcessAllBatches) + o.batch.processAllPendingBatches(&wgProcessAllBatches, MaxBatchSize) if !wantShutdown { wantShutdown = true } @@ -1265,8 +1262,8 @@ func (o *BatchOrchestrator) StartOrch() { } if !wantShutdown { if time.Since(lastFlush) > o.BatchInterval { - //log.Printf("[ORCHESTRATOR1] Timer triggered - processing all pending batches") - o.batch.processAllPendingBatches(&wgProcessAllBatches) + //log.Printf("[ORCHESTRATOR1] Timer triggered - processing all pending batches smaller than MaxBatchSize") + o.batch.processAllPendingBatches(&wgProcessAllBatches, MaxBatchSize-1) lastFlush = time.Now() } } @@ -1389,8 +1386,7 @@ fillQ: task.Mux.Unlock() return true } else { - log.Printf("[BATCH-BIG] Threshold exceeded for group '%s': %d articles (threshold: %d)", - *task.Newsgroup, batchCount, MaxBatchSize) + //log.Printf("[BATCH-BIG] Threshold exceeded for group '%s': %d articles (threshold: %d)", *task.Newsgroup, batchCount, MaxBatchSize) go o.batch.processNewsgroupBatch(task) totalQueued -= batchCount } @@ -1412,16 +1408,20 @@ fillQ: return haswork } -var BatchDividerChan = make(chan *models.Article, 100) +var BatchDividerChan = make(chan *models.Article, 1) + +// BatchDivider reads incoming articles and routes them to the appropriate per-newsgroup channel +// It also enforces the global MaxQueued limit to prevent overload +// Each newsgroup channel is created lazily on first use +// This runs as a single goroutine to avoid locking issues func (sq *SQ3batch) BatchDivider() { - //var TaskChans = make(map[*string]chan *models.Article) - var task *models.Article - var tasks *BatchTasks - var newsgroupPtr *string - //var BATCHchan chan *OverviewBatch + var tmpQueued, realQueue int + var maxQueue int = MaxQueued / 100 * 80 + var target int = MaxQueued / 100 * 20 for { - task = <-BatchDividerChan + var newsgroupPtr *string + task := <-BatchDividerChan if task == nil { log.Printf("[BATCH-DIVIDER] Received nil task?!") continue @@ -1434,13 +1434,36 @@ func (sq *SQ3batch) BatchDivider() { } //log.Printf("[BATCH-DIVIDER] Received task for group '%s'", *task.Newsgroup) //if TaskChans[newsgroupPtr] == nil { - tasks = sq.GetOrCreateTasksMapKey(*newsgroupPtr) + tasks := sq.GetOrCreateTasksMapKey(*newsgroupPtr) tasks.Mux.Lock() // Lazily create the per-group channel on first enqueue if tasks.BATCHchan == nil { tasks.BATCHchan = make(chan *models.Article, InitialBatchChannelSize) } tasks.Mux.Unlock() + if realQueue >= maxQueue { + log.Printf("[BATCH-DIVIDER] MaxQueued reached (%d), waiting to enqueue more (current Queue=%d, tmpQueued=%d)", MaxQueued, realQueue, tmpQueued) + + for { + time.Sleep(100 * time.Millisecond) + sq.GMux.RLock() + if sq.queued <= target { + realQueue = sq.queued + sq.GMux.RUnlock() + break + } + sq.GMux.RUnlock() + } + } tasks.BATCHchan <- task + tmpQueued++ + if tmpQueued >= MaxBatchSize { + sq.GMux.Lock() + //log.Printf("[BATCH-DIVIDER] Enqueued %d articles to group '%s' (current Queue=%d, tmpQueued=%d)", tmpQueued, *newsgroupPtr, realQueue, tmpQueued) + sq.queued += tmpQueued + tmpQueued = 0 + realQueue = sq.queued + sq.GMux.Unlock() + } } } diff --git a/internal/database/db_groupdbs.go b/internal/database/db_groupdbs.go index 44c25d4c..b55596b5 100644 --- a/internal/database/db_groupdbs.go +++ b/internal/database/db_groupdbs.go @@ -159,49 +159,14 @@ func (dbs *GroupDBs) IncrementWorkers() { } func (dbs *GroupDBs) Return(db *Database) { - dbs.mux.Lock() - dbs.Idle = time.Now() // Update idle time to now - dbs.Workers-- - dbs.mux.Unlock() - /* - if dbs.Workers < 0 { - log.Printf("Warning: Worker count went negative for group '%s'", dbs.Newsgroup) - return - } - //log.Printf("DEBUG: Return for group '%s': %d", dbs.Newsgroup, dbs.Workers) - - // Check if we need to close, but don't do it while holding dbs.mux - workerCount := dbs.Workers - ng := dbs.Newsgroup + if dbs != nil && db != nil { + dbs.mux.Lock() + dbs.Idle = time.Now() // Update idle time to now + dbs.Workers-- dbs.mux.Unlock() - - // Check openDBsNum with proper synchronization - db.MainMutex.RLock() - shouldClose := workerCount == 0 && db.openDBsNum >= MaxOpenDatabases // TODO HARDCODED - db.MainMutex.RUnlock() - - // If we need to close, acquire locks in the same order as GetGroupDBs to prevent deadlock - if shouldClose { - db.MainMutex.Lock() - dbs.mux.Lock() - // Double-check condition after re-acquiring locks - if dbs.Workers == 0 && db.openDBsNum >= MaxOpenDatabases { // TODO hardcoded limit - log.Printf("Closing group databases for '%s' due to no more open workers and high open DBs count (%d)", ng, db.openDBsNum) - err := dbs.Close() // Close DBs if no workers are left - if err != nil { - log.Printf("Failed to close group databases for '%s': %v", ng, err) - } else { - db.groupDBs[ng] = nil // Remove from groupDBs map - delete(db.groupDBs, ng) - dbs.DB = nil - db.openDBsNum-- - } - } - dbs.mux.Unlock() - db.MainMutex.Unlock() - } - */ - //dbs = nil + } else { + log.Printf("Warning: Attempted to return a nil db=%#v dbs=%#v", db, dbs) + } } func (db *GroupDBs) ExistsMsgIdInArticlesDB(messageID string) bool { diff --git a/internal/database/db_init.go b/internal/database/db_init.go index f6a4bf0c..be70e358 100644 --- a/internal/database/db_init.go +++ b/internal/database/db_init.go @@ -17,7 +17,7 @@ var GroupHashMap *GHmap // Global variable for group hash map var ENABLE_ARTICLE_CACHE = true var NNTP_AUTH_CACHE_TIME = 15 * time.Minute -var FETCH_MODE = false // set true in fetcher/main.go +var NO_CACHE_BOOT = false // set true in fetcher/main.go var SQLITE_cache_size = 2000 // 2000 pages or -2000 = 2 MB var SQLITE_busy_timeout = 30000 @@ -155,8 +155,8 @@ func OpenDatabase(dbconfig *DBConfig) (*Database, error) { } } db.StopChan = make(chan struct{}, 1) // Channel to signal shutdown (will get closed) - log.Printf("pugLeaf DB init config: %+v FETCH_MODE=%t", dbconfig, FETCH_MODE) - if !FETCH_MODE { + log.Printf("pugLeaf DB init config: %+v NO_CACHE_BOOT=%t", dbconfig, NO_CACHE_BOOT) + if !NO_CACHE_BOOT { db.SectionsCache = NewGroupSectionDBCache() db.MemThreadCache = NewMemCachedThreads() db.HierarchyCache = NewHierarchyCache() // Initialize hierarchy cache for fast browsing diff --git a/internal/database/db_post_queue.go b/internal/database/db_post_queue.go new file mode 100644 index 00000000..2fe55564 --- /dev/null +++ b/internal/database/db_post_queue.go @@ -0,0 +1,165 @@ +package database + +import ( + "fmt" + "log" + "strings" + "time" +) + +// PostQueueEntry represents a record in the post_queue table +type PostQueueEntry struct { + ID int64 `db:"id"` + NewsgroupID int64 `db:"newsgroup_id"` + MessageID string `db:"message_id"` // Added in migration 0016 + Created time.Time `db:"created"` + PostedToRemote bool `db:"posted_to_remote"` + InProcessing bool `db:"in_processing"` // Added in migration 0017 +} + +// InsertPostQueueEntry inserts a new entry into the post_queue table +// This is called when an article is first queued from the web interface +func (d *Database) InsertPostQueueEntry(newsgroupID int64, messageID string) error { + query := ` + INSERT INTO post_queue (newsgroup_id, message_id, created, posted_to_remote) + VALUES (?, ?, CURRENT_TIMESTAMP, 0) + ` + _, err := d.mainDB.Exec(query, newsgroupID, messageID) + if err != nil { + log.Printf("Database: Failed to insert post_queue entry: %v", err) + return err + } + + log.Printf("Database: Inserted post_queue entry msgId='%s'", messageID) + return nil +} + +// GetPendingPostQueueEntries retrieves entries that haven't been posted to remote servers and aren't being processed +func (d *Database) GetPendingPostQueueEntries(limit int) ([]PostQueueEntry, error) { + // Start a transaction to atomically select and mark as in_processing + tx, err := d.mainDB.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + // Select entries that are available for processing + query := ` + SELECT id, newsgroup_id, message_id, created, posted_to_remote, in_processing + FROM post_queue + WHERE posted_to_remote = 0 AND in_processing = 0 + ORDER BY created ASC LIMIT ? + ` + + rows, err := tx.Query(query, limit) + if err != nil { + return nil, err + } + defer rows.Close() + + var entries []PostQueueEntry + var ids []int64 + for rows.Next() { + var entry PostQueueEntry + err := rows.Scan(&entry.ID, &entry.NewsgroupID, &entry.MessageID, &entry.Created, &entry.PostedToRemote, &entry.InProcessing) + if err != nil { + return nil, err + } + entries = append(entries, entry) + ids = append(ids, entry.ID) + } + + // Mark all selected entries as in_processing + if len(ids) > 0 { + // Build the placeholders for the IN clause + placeholders := make([]string, len(ids)) + args := make([]interface{}, len(ids)) + for i, id := range ids { + placeholders[i] = "?" + args[i] = id + } + + updateQuery := fmt.Sprintf(`UPDATE post_queue SET in_processing = 1 WHERE id IN (%s)`, + strings.Join(placeholders, ",")) + + _, err = tx.Exec(updateQuery, args...) + if err != nil { + return nil, err + } + + // Update the entries to reflect the new state + for i := range entries { + entries[i].InProcessing = true + } + } + + // Commit the transaction + if err = tx.Commit(); err != nil { + return nil, err + } + + return entries, nil +} + +// MarkPostQueueAsPostedToRemote marks an entry as posted to remote servers and resets in_processing +func (d *Database) MarkPostQueueAsPostedToRemote(id int64) error { + query := `UPDATE post_queue SET posted_to_remote = 1, in_processing = 0 WHERE id = ?` + + _, err := d.mainDB.Exec(query, id) + if err != nil { + log.Printf("Database: Failed to mark post_queue entry %d as posted to remote: %v", id, err) + return err + } + + log.Printf("Database: Marked post_queue entry %d as posted to remote", id) + return nil +} + +// ResetPostQueueProcessing resets the in_processing flag for entries that failed processing +func (d *Database) ResetPostQueueProcessing(ids []int64) error { + if len(ids) == 0 { + return nil + } + + // Build the placeholders for the IN clause + placeholders := make([]string, len(ids)) + args := make([]interface{}, len(ids)) + for i, id := range ids { + placeholders[i] = "?" + args[i] = id + } + + query := fmt.Sprintf(`UPDATE post_queue SET in_processing = 0 WHERE id IN (%s)`, + strings.Join(placeholders, ",")) + + _, err := d.mainDB.Exec(query, args...) + if err != nil { + log.Printf("Database: Failed to reset in_processing for post_queue entries: %v", err) + return err + } + + log.Printf("Database: Reset in_processing flag for %d post_queue entries", len(ids)) + return nil +} + +// ResetAllPostQueueProcessing resets all in_processing flags - useful for cleanup on startup +func (d *Database) ResetAllPostQueueProcessing() error { + query := `UPDATE post_queue SET in_processing = 0 WHERE in_processing = 1` + + result, err := d.mainDB.Exec(query) + if err != nil { + log.Printf("Database: Failed to reset all in_processing flags: %v", err) + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + log.Printf("Database: Failed to get rows affected for reset all in_processing: %v", err) + return err + } + + if rowsAffected > 0 { + log.Printf("Database: Reset in_processing flag for %d stale post_queue entries", rowsAffected) + } + return nil +} diff --git a/internal/database/db_rescan.go b/internal/database/db_rescan.go index 5126551e..47d7a684 100644 --- a/internal/database/db_rescan.go +++ b/internal/database/db_rescan.go @@ -3,38 +3,12 @@ package database import ( "fmt" "log" + "strings" + "time" ) // RecoverDatabase attempts to recover the database by checking for missing articles and last_insert_ids mismatches -const batchSize = 25000 - -func (db *Database) Rescan(newsgroup string) error { - if newsgroup == "" { - return nil // Nothing to rescan - } - // first look into the maindb newsgroups table and get the latest numbers - latest, err := db.GetLatestArticleNumbers(newsgroup) - if err != nil { - return err - } - // open groupDBs - groupDB, err := db.GetGroupDBs(newsgroup) - if err != nil { - return err - } - defer groupDB.Return(db) - // Get the latest article number from the groupDB - latestArticle, err := db.GetLatestArticleNumberFromOverview(newsgroup) - if err != nil { - return err - } - // Compare with the latest from the mainDB - if latestArticle > latest[newsgroup] { - log.Printf("Found new articles in group '%s': %d (latest: %d)", newsgroup, latestArticle, latest[newsgroup]) - // TODO: Handle new articles (e.g., fetch and insert into mainDB) - } - return nil -} +var RescanBatchSize int64 = 25000 func (db *Database) GetLatestArticleNumberFromOverview(newsgroup string) (int64, error) { // Since overview table is unified with articles, query articles table instead @@ -192,13 +166,13 @@ func (db *Database) findMissingArticles(groupDB *GroupDBs, maxArticleNum int64) var offset int64 = 0 var totalProcessed int64 = 0 - log.Printf("Checking for missing articles in batches of %d (max article: %d)", batchSize, maxArticleNum) + log.Printf("Checking for missing articles in batches of %d (max article: %d)", RescanBatchSize, maxArticleNum) for offset < maxArticleNum { // Get batch of article numbers rows, err := retryableQuery(groupDB.DB, "SELECT article_num FROM articles WHERE article_num > ? ORDER BY article_num LIMIT ?", - offset, batchSize) + offset, RescanBatchSize) if err != nil { log.Printf("Error fetching article batch starting at %d: %v", offset, err) break @@ -233,7 +207,7 @@ func (db *Database) findMissingArticles(groupDB *GroupDBs, maxArticleNum int64) totalProcessed += int64(len(batchArticles)) // Progress reporting for large groups - if totalProcessed%100000 == 0 { + if totalProcessed%10000 == 0 { log.Printf("Processed %d articles, found %d missing so far", totalProcessed, len(missing)) } } @@ -246,7 +220,7 @@ func (db *Database) findMissingArticles(groupDB *GroupDBs, maxArticleNum int64) func (db *Database) findOrphanedThreads(groupDB *GroupDBs) []int64 { var orphaned []int64 - log.Printf("Building article index in batches of %d", batchSize) + log.Printf("Building article index in batches of %d", RescanBatchSize) // Build a map of existing article numbers using batched processing articleNums := make(map[int64]bool) @@ -257,13 +231,13 @@ func (db *Database) findOrphanedThreads(groupDB *GroupDBs) []int64 { // Get batch of article numbers rows, err := retryableQuery(groupDB.DB, "SELECT article_num FROM articles WHERE article_num > ? ORDER BY article_num LIMIT ?", - offset, batchSize) + offset, RescanBatchSize) if err != nil { log.Printf("Error fetching article batch for orphan check starting at %d: %v", offset, err) return orphaned } - var batchCount int + var batchCount int64 var lastArticle int64 for rows.Next() { var num int64 @@ -284,11 +258,11 @@ func (db *Database) findOrphanedThreads(groupDB *GroupDBs) []int64 { offset = lastArticle // Progress reporting for large groups - if totalArticles%100000 == 0 { + if totalArticles%10000 == 0 { log.Printf("Indexed %d articles for orphan detection", totalArticles) } - if batchCount < batchSize { + if batchCount < RescanBatchSize { break // Last batch } } @@ -303,13 +277,13 @@ func (db *Database) findOrphanedThreads(groupDB *GroupDBs) []int64 { // Get batch of distinct root_article numbers from threads table rows, err := retryableQuery(groupDB.DB, "SELECT DISTINCT root_article FROM threads WHERE root_article > ? ORDER BY root_article LIMIT ?", - offset, batchSize) + offset, RescanBatchSize) if err != nil { log.Printf("Error fetching thread batch for orphan check starting at %d: %v", offset, err) return orphaned } - var batchCount int + var batchCount int64 var lastRoot int64 for rows.Next() { var rootArticle int64 @@ -337,7 +311,7 @@ func (db *Database) findOrphanedThreads(groupDB *GroupDBs) []int64 { log.Printf("Checked %d thread roots, found %d orphaned so far", totalThreads, len(orphaned)) } - if batchCount < batchSize { + if batchCount < RescanBatchSize { break // Last batch } } @@ -397,78 +371,564 @@ func (report *ConsistencyReport) PrintReport() { fmt.Printf("============================================\n\n") } -/* CODE REFERENCE - -type Article struct { - GetDataFunc func(what string, group string) string `json:"-" db:"-"` - RWMutex sync.RWMutex `json:"-" db:"-"` - ArticleNum int64 `json:"article_num" db:"article_num"` - MessageID string `json:"message_id" db:"message_id"` - Subject string `json:"subject" db:"subject"` - FromHeader string `json:"from_header" db:"from_header"` - DateSent time.Time `json:"date_sent" db:"date_sent"` - DateString string `json:"date_string" db:"date_string"` - References string `json:"references" db:"references"` - Bytes int `json:"bytes" db:"bytes"` - Lines int `json:"lines" db:"lines"` - ReplyCount int `json:"reply_count" db:"reply_count"` - HeadersJSON string `json:"headers_json" db:"headers_json"` - BodyText string `json:"body_text" db:"body_text"` - Path string `json:"path" db:"path"` // headers network path - ImportedAt time.Time `json:"imported_at" db:"imported_at"` - Sanitized bool `json:"-" db:"-"` - MsgIdItem *history.MessageIdItem `json:"-" db:"-"` // Cached MessageIdItem for history lookups +const query_RebuildThreadsFromScratch1 = "SELECT COUNT(*) FROM articles" +const query_RebuildThreadsFromScratch2 = "SELECT COUNT(*) FROM threads" +const query_RebuildThreadsFromScratch3 = "DELETE FROM %s" +const query_RebuildThreadsFromScratch4 = "DELETE FROM sqlite_sequence WHERE name = 'threads'" +const query_RebuildThreadsFromScratch5 = "SELECT article_num, message_id FROM articles ORDER BY article_num LIMIT ? OFFSET ?" + +// RebuildThreadsFromScratch completely rebuilds all thread relationships for a newsgroup +// This function deletes all existing threads and rebuilds them from article 1 based on message references +func (db *Database) RebuildThreadsFromScratch(newsgroup string, verbose bool) (*ThreadRebuildReport, error) { + report := &ThreadRebuildReport{ + Newsgroup: newsgroup, + StartTime: time.Now(), + Errors: []string{}, + } + + if verbose { + log.Printf("RebuildThreadsFromScratch: Starting complete thread rebuild for newsgroup '%s'", newsgroup) + } + + // Get group database + groupDB, err := db.GetGroupDBs(newsgroup) + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to get group database: %v", err)) + return report, err + } + defer groupDB.Return(db) + + // Get total article count + err = retryableQueryRowScan(groupDB.DB, query_RebuildThreadsFromScratch1, []interface{}{}, &report.TotalArticles) + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to get article count: %v", err)) + return report, err + } + + if report.TotalArticles == 0 { + if verbose { + log.Printf("RebuildThreadsFromScratch: No articles found in newsgroup '%s', nothing to rebuild", newsgroup) + } + report.ThreadsRebuilt = 0 + report.EndTime = time.Now() + return report, nil + } + + if verbose { + log.Printf("RebuildThreadsFromScratch: Found %d articles to process", report.TotalArticles) + } + + // Step 1: Clear existing thread data + if verbose { + log.Printf("RebuildThreadsFromScratch: Clearing existing thread data...") + } + + tx, err := groupDB.DB.Begin() + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to begin cleanup transaction: %v", err)) + return report, err + } + defer tx.Rollback() + + // Get count of existing threads for reporting + var existingThreads int64 + tx.QueryRow(query_RebuildThreadsFromScratch2).Scan(&existingThreads) + report.ThreadsDeleted = existingThreads + + // Clear thread-related tables in dependency order + tables := []string{"tree_stats", "cached_trees", "thread_cache", "threads"} + for _, table := range tables { + _, err = tx.Exec(fmt.Sprintf(query_RebuildThreadsFromScratch3, table)) + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to clear table %s: %v", table, err)) + return report, err + } + } + + // Reset auto-increment for threads table + _, err = tx.Exec(query_RebuildThreadsFromScratch4) + if err != nil { + // Non-critical error + if verbose { + log.Printf("RebuildThreadsFromScratch: Warning - could not reset auto-increment for threads: %v", err) + } + } + + err = tx.Commit() + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to commit cleanup transaction: %v", err)) + return report, err + } + + if verbose { + log.Printf("RebuildThreadsFromScratch: Cleared %d existing thread entries", existingThreads) + } + + // Step 2: Build message-ID to article-number mapping + if verbose { + log.Printf("RebuildThreadsFromScratch: Building message-ID mapping...") + } + + msgIDToArticleNum := make(map[string]int64) + var offset int64 = 0 + + for offset < report.TotalArticles { + currentBatchSize := RescanBatchSize + if offset+RescanBatchSize > report.TotalArticles { + currentBatchSize = report.TotalArticles - offset + } + + // Load batch of article mappings + rows, err := retryableQuery(groupDB.DB, query_RebuildThreadsFromScratch5, currentBatchSize, offset) + + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to query articles batch: %v", err)) + return report, err + } + + for rows.Next() { + var articleNum int64 + var messageID string + if err := rows.Scan(&articleNum, &messageID); err != nil { + rows.Close() + report.Errors = append(report.Errors, fmt.Sprintf("Failed to scan article mapping: %v", err)) + return report, err + } + msgIDToArticleNum[messageID] = articleNum + } + rows.Close() + + offset += int64(currentBatchSize) + + if verbose && offset%1000 == 0 { + log.Printf("RebuildThreadsFromScratch: Built message-ID mapping: %d/%d articles", offset, report.TotalArticles) + } + } + + if verbose { + log.Printf("RebuildThreadsFromScratch: Message-ID mapping complete: %d entries", len(msgIDToArticleNum)) + } + + // Step 3: Process articles in batches to build thread relationships + if verbose { + log.Printf("RebuildThreadsFromScratch: Building thread relationships...") + } + + offset = 0 + for offset < report.TotalArticles { + currentBatchSize := RescanBatchSize + if offset+RescanBatchSize > report.TotalArticles { + currentBatchSize = report.TotalArticles - offset + } + + threadsBuilt, err := db.processThreadBatch(groupDB, msgIDToArticleNum, offset, currentBatchSize, verbose) + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("Failed to process thread batch at offset %d: %v", offset, err)) + return report, err + } + + report.ThreadsRebuilt += int64(threadsBuilt) + offset += int64(currentBatchSize) + + if verbose && offset%1000 == 0 { + log.Printf("RebuildThreadsFromScratch: Threading progress: %d/%d articles processed, %d threads built", + offset, report.TotalArticles, report.ThreadsRebuilt) + } + } + + report.EndTime = time.Now() + report.Duration = report.EndTime.Sub(report.StartTime) + + if verbose { + log.Printf("RebuildThreadsFromScratch: Completed successfully for newsgroup '%s'", newsgroup) + log.Printf(" - Articles processed: %d", report.TotalArticles) + log.Printf(" - Threads deleted: %d", report.ThreadsDeleted) + log.Printf(" - Threads rebuilt: %d", report.ThreadsRebuilt) + log.Printf(" - Duration: %d ms", report.Duration.Milliseconds()) + } + msgIDToArticleNum = nil + return report, nil } -// Newsgroup represents a subscribed newsgroup -type Newsgroup struct { - ID int `json:"id" db:"id"` - Name string `json:"name" db:"name"` - Active bool `json:"active" db:"active"` - Description string `json:"description" db:"description"` - LastArticle int64 `json:"last_article" db:"last_article"` - MessageCount int64 `json:"message_count" db:"message_count"` - ExpiryDays int `json:"expiry_days" db:"expiry_days"` - MaxArticles int `json:"max_articles" db:"max_articles"` - MaxArtSize int `json:"max_art_size" db:"max_art_size"` - // NNTP-specific fields - HighWater int `json:"high_water" db:"high_water"` - LowWater int `json:"low_water" db:"low_water"` - Status string `json:"status" db:"status"` - CreatedAt time.Time `json:"created_at" db:"created_at"` - UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +const query_processThreadBatch1 = ` + SELECT article_num, message_id, "references", date_sent + FROM articles + ORDER BY article_num + LIMIT ? OFFSET ? + ` +const query_processThreadBatch2 = "INSERT INTO threads (root_article, parent_article, child_article, depth, thread_order) VALUES (?, ?, ?, 0, 0)" +const query_processThreadBatch3 = "INSERT INTO threads (root_article, parent_article, child_article, depth, thread_order) VALUES (?, ?, ?, ?, 0)" + +// processThreadBatch processes a batch of articles to build thread relationships +// Based on the actual threading system: only ROOT articles go in threads table, replies only update thread_cache +func (db *Database) processThreadBatch(groupDB *GroupDBs, msgIDToArticleNum map[string]int64, offset, batchSize int64, verbose bool) (int, error) { + // Get batch of articles with their references and dates + rows, err := retryableQuery(groupDB.DB, query_processThreadBatch1, batchSize, offset) + if err != nil { + return 0, fmt.Errorf("failed to query articles: %w", err) + } + defer rows.Close() + + // Separate roots and replies for processing + var threadRoots []struct { + articleNum int64 + dateSent time.Time + } + var threadReplies []struct { + articleNum int64 + parentNum int64 + rootNum int64 + dateSent time.Time + depth int + } + + // Process each article to determine if it's a root or reply + for rows.Next() { + var articleNum int64 + var messageID, references string + var dateSent time.Time + + err := rows.Scan(&articleNum, &messageID, &references, &dateSent) + if err != nil { + if verbose { + log.Printf("processThreadBatch: Error scanning article: %v", err) + } + continue + } + + refs := db.parseReferences(references) + + if len(refs) == 0 { + // This is a thread root - will be inserted into threads table + threadRoots = append(threadRoots, struct { + articleNum int64 + dateSent time.Time + }{articleNum, dateSent}) + } else { + // This is a reply - find its immediate parent and thread root + var parentArticleNum, rootArticleNum int64 + depth := 0 + + // Find immediate parent (last reference that exists) + for i := len(refs) - 1; i >= 0; i-- { + if parentNum, exists := msgIDToArticleNum[refs[i]]; exists { + parentArticleNum = parentNum + depth = i + 1 // Depth based on position in references + break + } + } + + // Find thread root (first reference that exists) + for i := 0; i < len(refs); i++ { + if rootNum, exists := msgIDToArticleNum[refs[i]]; exists { + rootArticleNum = rootNum + break + } + } + + // If no root found in references, the immediate parent becomes the root + if rootArticleNum == 0 { + rootArticleNum = parentArticleNum + } + + if parentArticleNum > 0 { + threadReplies = append(threadReplies, struct { + articleNum int64 + parentNum int64 + rootNum int64 + dateSent time.Time + depth int + }{articleNum, parentArticleNum, rootArticleNum, dateSent, depth}) + } + } + } + + if err = rows.Err(); err != nil { + return 0, fmt.Errorf("error iterating articles: %w", err) + } + + threadsBuilt := 0 + + // Step 1: Insert thread ROOTS into threads table + if len(threadRoots) > 0 { + tx, err := groupDB.DB.Begin() + if err != nil { + return 0, fmt.Errorf("failed to begin threads transaction: %w", err) + } + defer tx.Rollback() + + threadStmt, err := tx.Prepare(query_processThreadBatch2) + if err != nil { + return 0, fmt.Errorf("failed to prepare thread insert statement: %w", err) + } + defer threadStmt.Close() + + for _, root := range threadRoots { + // For thread roots: root_article = child_article, parent_article = NULL + _, err = threadStmt.Exec(root.articleNum, nil, root.articleNum) + if err != nil { + if verbose { + log.Printf("processThreadBatch: Failed to insert thread root for article %d: %v", root.articleNum, err) + } + continue + } + threadsBuilt++ + } + + if err := tx.Commit(); err != nil { + return threadsBuilt, fmt.Errorf("failed to commit threads transaction: %w", err) + } + + // Step 2: Initialize thread_cache for roots + for _, root := range threadRoots { + err := db.initializeThreadCacheSimple(groupDB, root.articleNum, root.dateSent) + if err != nil { + if verbose { + log.Printf("processThreadBatch: Failed to initialize thread cache for root %d: %v", root.articleNum, err) + } + // Don't fail the whole operation for cache errors + } + } + } + + // Step 3: Insert REPLIES into threads table + if len(threadReplies) > 0 { + tx, err := groupDB.DB.Begin() + if err != nil { + return threadsBuilt, fmt.Errorf("failed to begin replies transaction: %w", err) + } + defer tx.Rollback() + + replyStmt, err := tx.Prepare(query_processThreadBatch3) + if err != nil { + return threadsBuilt, fmt.Errorf("failed to prepare reply insert statement: %w", err) + } + defer replyStmt.Close() + + repliesBuilt := 0 + for _, reply := range threadReplies { + // For replies: root_article = thread root, parent_article = immediate parent, child_article = this article + _, err = replyStmt.Exec(reply.rootNum, reply.parentNum, reply.articleNum, reply.depth) + if err != nil { + if verbose { + log.Printf("processThreadBatch: Failed to insert reply for article %d: %v", reply.articleNum, err) + } + continue + } + repliesBuilt++ + } + + if err := tx.Commit(); err != nil { + return threadsBuilt, fmt.Errorf("failed to commit replies transaction: %w", err) + } + + if verbose && repliesBuilt > 0 { + log.Printf("processThreadBatch: Inserted %d replies into threads table", repliesBuilt) + } + + // Step 4: Update thread_cache for replies (build cache updates from replies) + threadCacheUpdates := make(map[int64][]int64) // Changed to map[rootID][]childArticleNums + for _, reply := range threadReplies { + threadCacheUpdates[reply.rootNum] = append(threadCacheUpdates[reply.rootNum], reply.articleNum) + } + + if len(threadCacheUpdates) > 0 { + if err := db.updateThreadCacheWithChildren(groupDB, threadCacheUpdates, verbose); err != nil { + if verbose { + log.Printf("processThreadBatch: Failed to update thread cache: %v", err) + } + // Don't fail the whole operation for cache errors + } + } + } + + return threadsBuilt, nil +} + +const query_initializeThreadCacheSimple1 = ` + INSERT OR REPLACE INTO thread_cache ( + thread_root, root_date, message_count, child_articles, last_child_number, last_activity + ) VALUES (?, ?, 1, '', ?, ?) + ` + +// initializeThreadCacheSimple initializes thread cache for a root article +func (db *Database) initializeThreadCacheSimple(groupDB *GroupDBs, threadRoot int64, rootDate time.Time) error { + // Validate root date - skip obvious future posts + now := time.Now().UTC() + futureLimit := now.Add(25 * time.Hour) + + if rootDate.UTC().After(futureLimit) { + log.Printf("initializeThreadCacheSimple: Skipping thread root %d with future date %v", + threadRoot, rootDate.Format("2006-01-02 15:04:05")) + // Use current time as fallback for obvious future posts + rootDate = now + } + + _, err := retryableExec(groupDB.DB, query_initializeThreadCacheSimple1, + threadRoot, + rootDate.UTC().Format("2006-01-02 15:04:05"), + threadRoot, // last_child_number starts as the root itself + rootDate.UTC().Format("2006-01-02 15:04:05"), + ) + + if err != nil { + return fmt.Errorf("failed to initialize thread cache for root %d: %w", threadRoot, err) + } + + return nil } -type Overview struct { - ArticleNum int64 `json:"article_num" db:"article_num"` - Subject string `json:"subject" db:"subject"` - FromHeader string `json:"from_header" db:"from_header"` - DateSent time.Time `json:"date_sent" db:"date_sent"` - DateString string `json:"date_string" db:"date_string"` - MessageID string `json:"message_id" db:"message_id"` - References string `json:"references" db:"references"` - Bytes int `json:"bytes" db:"bytes"` - Lines int `json:"lines" db:"lines"` - ReplyCount int `json:"reply_count" db:"reply_count"` - Downloaded int `json:"downloaded" db:"downloaded"` // 0 = not downloaded, 1 = downloaded - Sanitized bool `json:"-" db:"-"` +const query_updateThreadCacheWithChildren1 = "SELECT message_count, child_articles FROM thread_cache WHERE thread_root = ?" +const query_updateThreadCacheWithChildren2 = ` + UPDATE thread_cache + SET child_articles = ?, + message_count = ? + WHERE thread_root = ? + ` + +// updateThreadCacheWithChildren updates the thread_cache table with child article lists +func (db *Database) updateThreadCacheWithChildren(groupDB *GroupDBs, rootUpdates map[int64][]int64, verbose bool) error { + if len(rootUpdates) == 0 { + return nil + } + + tx, err := groupDB.DB.Begin() + if err != nil { + return fmt.Errorf("failed to begin thread cache transaction: %w", err) + } + defer tx.Rollback() + + // Update each thread root's cache + for rootArticle, childArticleNums := range rootUpdates { + // Build comma-separated child articles list + childArticlesStr := "" + if len(childArticleNums) > 0 { + childStrs := make([]string, len(childArticleNums)) + for i, num := range childArticleNums { + childStrs[i] = fmt.Sprintf("%d", num) + } + childArticlesStr = strings.Join(childStrs, ",") + } + + // Get current thread cache data + var currentCount int + var currentChildren string + err := tx.QueryRow(query_updateThreadCacheWithChildren1, rootArticle).Scan(¤tCount, ¤tChildren) + if err != nil { + if verbose { + log.Printf("updateThreadCacheWithChildren: No thread cache entry for root %d, skipping", rootArticle) + } + continue + } + + // Merge with existing children (in case we're processing in batches) + var allChildren []string + if currentChildren != "" { + allChildren = append(allChildren, strings.Split(currentChildren, ",")...) + } + if childArticlesStr != "" { + allChildren = append(allChildren, strings.Split(childArticlesStr, ",")...) + } + + // Remove duplicates and build final child list + childMap := make(map[string]bool) + for _, child := range allChildren { + if child != "" { + childMap[child] = true + } + } + + var finalChildren []string + for child := range childMap { + finalChildren = append(finalChildren, child) + } + + finalChildrenStr := strings.Join(finalChildren, ",") + newMessageCount := 1 + len(finalChildren) // 1 for root + replies + + // Update thread_cache with child articles list and correct message count + _, err = tx.Exec(query_updateThreadCacheWithChildren2, finalChildrenStr, newMessageCount, rootArticle) + + if err != nil { + if verbose { + log.Printf("updateThreadCacheWithChildren: Failed to update cache for root %d: %v", rootArticle, err) + } + continue + } + + if verbose { + log.Printf("updateThreadCacheWithChildren: Updated root %d with %d replies: %s", + rootArticle, len(finalChildren), finalChildrenStr) + } + } + + return tx.Commit() } -// ForumThread represents a complete thread with root article and replies -type ForumThread struct { - RootArticle *Overview `json:"thread_root"` // The original post - Replies []*Overview `json:"replies"` // All replies in flat list - MessageCount int `json:"message_count"` // Total messages in thread - LastActivity time.Time `json:"last_activity"` // Most recent activity +// parseReferences parses the references header into individual message IDs +func (db *Database) parseReferences(refs string) []string { + if refs == "" { + return []string{} + } + + // Use strings.Fields() for robust whitespace handling + fields := strings.Fields(refs) + + var cleanRefs []string + for _, ref := range fields { + ref = strings.TrimSpace(ref) + if ref != "" && strings.HasPrefix(ref, "<") && strings.HasSuffix(ref, ">") { + cleanRefs = append(cleanRefs, ref) + } + } + + return cleanRefs +} + +// ThreadRebuildReport represents the results of a thread rebuild operation +type ThreadRebuildReport struct { + Newsgroup string + TotalArticles int64 + ThreadsDeleted int64 + ThreadsRebuilt int64 + FutureDatesFixed int64 // New: count of future dates corrected + Errors []string + StartTime time.Time + EndTime time.Time + Duration time.Duration } -// Thread represents a parent/child relationship for threading -type Thread struct { - ID int `json:"id" db:"id"` - RootArticle int64 `json:"root_article" db:"root_article"` - ParentArticle *int64 `json:"parent_article" db:"parent_article"` // Pointer for NULL values - ChildArticle int64 `json:"child_article" db:"child_article"` - Depth int `json:"depth" db:"depth"` - ThreadOrder int `json:"thread_order" db:"thread_order"` +// PrintReport prints a human-readable thread rebuild report +func (report *ThreadRebuildReport) PrintReport() { + fmt.Printf("\n=== Thread Rebuild Report for '%s' ===\n", report.Newsgroup) + + if len(report.Errors) > 0 { + fmt.Printf("ERRORS:\n") + for _, err := range report.Errors { + fmt.Printf(" - %s\n", err) + } + fmt.Printf("\n") + } + if report.TotalArticles > 0 { + fmt.Printf("Articles processed: %d\n", report.TotalArticles) + } + if report.ThreadsDeleted > 0 || report.ThreadsRebuilt > 0 { + fmt.Printf("Threads deleted/rebuilt: %d/%d\n", report.ThreadsDeleted, report.ThreadsRebuilt) + } + if report.FutureDatesFixed > 0 { + fmt.Printf("Future dates fixed: %d\n", report.FutureDatesFixed) + } + fmt.Printf("Duration: %d ms\n", report.Duration.Milliseconds()) + + if len(report.Errors) == 0 { + fmt.Printf("\n✅ Thread rebuild completed successfully.\n") + if report.FutureDatesFixed > 0 { + fmt.Printf("💡 Fixed %d future date issues during rebuild.\n", report.FutureDatesFixed) + } + } else { + fmt.Printf("\n❌ Thread rebuild completed with errors.\n") + } + fmt.Printf("===========================================\n\n") } -*/ + +/* CODE REFERENCES: internal/database/models.go */ diff --git a/internal/database/migrations/0011_main_add_posting_enabled.sql b/internal/database/migrations/0011_main_add_posting_enabled.sql new file mode 100644 index 00000000..02c8e212 --- /dev/null +++ b/internal/database/migrations/0011_main_add_posting_enabled.sql @@ -0,0 +1,14 @@ +-- go-pugleaf: Add posting field to providers table +-- Migration: 0011 +-- Purpose: Allow enabling/disabling posting capability per provider + +PRAGMA foreign_keys = ON; + +-- Add posting field to providers table +ALTER TABLE providers ADD COLUMN posting INTEGER NOT NULL DEFAULT 0 CHECK(posting IN (0, 1)); + +-- Create index for posting field for better query performance +CREATE INDEX IF NOT EXISTS idx_providers_posting ON providers(posting); + +-- Optional: Update any existing providers to have posting disabled by default +-- (This is already handled by the DEFAULT 0 in the column definition) diff --git a/internal/database/migrations/0012_main_add_proxy_support.sql b/internal/database/migrations/0012_main_add_proxy_support.sql new file mode 100644 index 00000000..ed9ab5e9 --- /dev/null +++ b/internal/database/migrations/0012_main_add_proxy_support.sql @@ -0,0 +1,25 @@ +-- go-pugleaf: Add proxy support for NNTP providers +-- Created: 2025-09-12 +-- Adds proxy configuration fields to providers table for Tor and proxy support + +PRAGMA foreign_keys = ON; + +-- Add proxy configuration fields to providers table +ALTER TABLE providers ADD COLUMN proxy_enabled BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE providers ADD COLUMN proxy_type TEXT NOT NULL DEFAULT ''; +ALTER TABLE providers ADD COLUMN proxy_host TEXT DEFAULT ''; +ALTER TABLE providers ADD COLUMN proxy_port INTEGER DEFAULT 0; +ALTER TABLE providers ADD COLUMN proxy_username TEXT DEFAULT ''; +ALTER TABLE providers ADD COLUMN proxy_password TEXT DEFAULT ''; + +-- Add check constraint for valid proxy types +-- Note: SQLite doesn't support adding constraints to existing tables, so we'll handle validation in Go + +-- Create index for proxy-enabled providers +CREATE INDEX IF NOT EXISTS idx_providers_proxy_enabled ON providers(proxy_enabled); + +-- Insert default configuration values for proxy settings +INSERT OR IGNORE INTO config (key, value) VALUES ('default_tor_proxy_host', '127.0.0.1'); +INSERT OR IGNORE INTO config (key, value) VALUES ('default_tor_proxy_port', '9050'); +INSERT OR IGNORE INTO config (key, value) VALUES ('proxy_connect_timeout', '30'); +INSERT OR IGNORE INTO config (key, value) VALUES ('proxy_fallback_enabled', 'false'); diff --git a/internal/database/migrations/0013_main_add_post_queue.sql b/internal/database/migrations/0013_main_add_post_queue.sql new file mode 100644 index 00000000..a1c3885c --- /dev/null +++ b/internal/database/migrations/0013_main_add_post_queue.sql @@ -0,0 +1,28 @@ +-- go-pugleaf: Add post_queue table for tracking posted articles from web interface +-- Created: 2025-09-12 +-- This table tracks articles posted through the web interface to manage posting to remote servers + +PRAGMA foreign_keys = ON; + +-- Create post_queue table to track articles posted from web interface +CREATE TABLE IF NOT EXISTS post_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + artnum INTEGER NOT NULL, + newsgroup_id INTEGER NOT NULL, + created DATETIME DEFAULT CURRENT_TIMESTAMP, + posted_to_remote INTEGER NOT NULL DEFAULT 0 CHECK (posted_to_remote IN (0, 1)), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (newsgroup_id) REFERENCES newsgroups(id) ON DELETE CASCADE, + UNIQUE(newsgroup_id, artnum) +); + +-- Create indexes for performance +CREATE INDEX IF NOT EXISTS idx_post_queue_user_id ON post_queue(user_id); +CREATE INDEX IF NOT EXISTS idx_post_queue_newsgroup_id ON post_queue(newsgroup_id); +CREATE INDEX IF NOT EXISTS idx_post_queue_artnum ON post_queue(artnum); +CREATE INDEX IF NOT EXISTS idx_post_queue_posted_to_remote ON post_queue(posted_to_remote); +CREATE INDEX IF NOT EXISTS idx_post_queue_created ON post_queue(created); + +-- Create composite index for common queries +CREATE INDEX IF NOT EXISTS idx_post_queue_newsgroup_posted ON post_queue(newsgroup_id, posted_to_remote); diff --git a/internal/database/migrations/0014_main_add_nntp_hostname_config.sql b/internal/database/migrations/0014_main_add_nntp_hostname_config.sql new file mode 100644 index 00000000..b4dfda5e --- /dev/null +++ b/internal/database/migrations/0014_main_add_nntp_hostname_config.sql @@ -0,0 +1,9 @@ +-- go-pugleaf: Add NNTP hostname configuration +-- Created: 2025-09-12 +-- Adds LocalNNTPHostname configuration to support processor hostname validation + +PRAGMA foreign_keys = ON; + +-- Insert default configuration value for NNTP hostname +-- Empty default value - will be set by admin or fail validation if not configured +INSERT OR IGNORE INTO config (key, value) VALUES ('local_nntp_hostname', ''); diff --git a/internal/database/migrations/0015_main_add_web_post_max_article_size_config.sql b/internal/database/migrations/0015_main_add_web_post_max_article_size_config.sql new file mode 100644 index 00000000..4f0d51e4 --- /dev/null +++ b/internal/database/migrations/0015_main_add_web_post_max_article_size_config.sql @@ -0,0 +1,5 @@ +-- Migration: Add WebPostMaxArticleSize config key +-- This sets the maximum size limit for articles posted via web interface + +INSERT INTO config (key, value) +VALUES ('WebPostMaxArticleSize', '32768'); diff --git a/internal/database/migrations/0016_main_add_message_id_to_post_queue.sql b/internal/database/migrations/0016_main_add_message_id_to_post_queue.sql new file mode 100644 index 00000000..232f0eec --- /dev/null +++ b/internal/database/migrations/0016_main_add_message_id_to_post_queue.sql @@ -0,0 +1,22 @@ +-- go-pugleaf: Add message_id field to post_queue table +-- Created: 2025-09-13 +-- This migration adds a message_id field to track articles before they get article numbers + +-- Create post_queue table to track articles posted from web interface +DROP TABLE IF EXISTS post_queue; + +CREATE TABLE IF NOT EXISTS post_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + newsgroup_id INTEGER NOT NULL, + message_id TEXT, + created DATETIME DEFAULT CURRENT_TIMESTAMP, + posted_to_remote INTEGER NOT NULL DEFAULT 0 CHECK (posted_to_remote IN (0, 1)), + FOREIGN KEY (newsgroup_id) REFERENCES newsgroups(id) ON DELETE CASCADE +); + +-- Create indexes for performance +CREATE INDEX IF NOT EXISTS idx_post_queue_newsgroup_id ON post_queue(newsgroup_id); +CREATE INDEX IF NOT EXISTS idx_post_queue_posted_to_remote ON post_queue(posted_to_remote); +CREATE INDEX IF NOT EXISTS idx_post_queue_created ON post_queue(created); +CREATE INDEX IF NOT EXISTS idx_post_queue_message_id ON post_queue(message_id); +CREATE INDEX IF NOT EXISTS idx_post_queue_newsgroup_posted ON post_queue(newsgroup_id, posted_to_remote); \ No newline at end of file diff --git a/internal/database/migrations/0017_main_add_in_processing_to_post_queue.sql b/internal/database/migrations/0017_main_add_in_processing_to_post_queue.sql new file mode 100644 index 00000000..b1814f64 --- /dev/null +++ b/internal/database/migrations/0017_main_add_in_processing_to_post_queue.sql @@ -0,0 +1,14 @@ +-- go-pugleaf: Add in_processing field to post_queue table +-- Created: 2025-09-13 +-- This migration adds an in_processing field to prevent multiple processes from working on the same articles + +PRAGMA foreign_keys = ON; + +-- Add in_processing field to post_queue table +ALTER TABLE post_queue ADD COLUMN in_processing INTEGER NOT NULL DEFAULT 0 CHECK (in_processing IN (0, 1)); + +-- Create index for the new field for performance +CREATE INDEX IF NOT EXISTS idx_post_queue_in_processing ON post_queue(in_processing); + +-- Create composite index for common queries (not posted and not in processing) +CREATE INDEX IF NOT EXISTS idx_post_queue_available ON post_queue(posted_to_remote, in_processing) WHERE posted_to_remote = 0 AND in_processing = 0; diff --git a/internal/database/queries.go b/internal/database/queries.go index 84d88eba..77cb1a97 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -48,15 +48,17 @@ func parseDateString(dateStr string) time.Time { // --- Main DB Queries --- // AddProvider adds a new provider to the main database -const query_AddProvider = `INSERT INTO providers (name, grp, host, port, ssl, username, password, max_conns, enabled, priority, max_art_size) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` +const query_AddProvider = `INSERT INTO providers (name, grp, host, port, ssl, username, password, max_conns, enabled, priority, max_art_size, posting, proxy_enabled, proxy_type, proxy_host, proxy_port, proxy_username, proxy_password) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` func (db *Database) AddProvider(provider *models.Provider) error { _, err := retryableExec(db.mainDB, query_AddProvider, provider.Name, provider.Grp, provider.Host, provider.Port, provider.SSL, provider.Username, provider.Password, provider.MaxConns, provider.Enabled, provider.Priority, - provider.MaxArtSize) + provider.MaxArtSize, provider.Posting, provider.ProxyEnabled, + provider.ProxyType, provider.ProxyHost, provider.ProxyPort, + provider.ProxyUsername, provider.ProxyPassword) if err != nil { return fmt.Errorf("failed to add provider %s: %w", provider.Name, err) } @@ -74,7 +76,8 @@ func (db *Database) DeleteProvider(id int) error { const query_SetProvider = `UPDATE providers SET grp = ?, host = ?, port = ?, ssl = ?, username = ?, password = ?, - max_conns = ?, enabled = ?, priority = ?, max_art_size = ? + max_conns = ?, enabled = ?, priority = ?, max_art_size = ?, posting = ?, + proxy_enabled = ?, proxy_type = ?, proxy_host = ?, proxy_port = ?, proxy_username = ?, proxy_password = ? WHERE id = ?` func (db *Database) SetProvider(provider *models.Provider) error { @@ -82,7 +85,9 @@ func (db *Database) SetProvider(provider *models.Provider) error { provider.Grp, provider.Host, provider.Port, provider.SSL, provider.Username, provider.Password, provider.MaxConns, provider.Enabled, provider.Priority, - provider.MaxArtSize, provider.ID) + provider.MaxArtSize, provider.Posting, provider.ProxyEnabled, + provider.ProxyType, provider.ProxyHost, provider.ProxyPort, + provider.ProxyUsername, provider.ProxyPassword, provider.ID) if err != nil { return fmt.Errorf("failed to update provider %d: %w", provider.ID, err) } @@ -90,7 +95,7 @@ func (db *Database) SetProvider(provider *models.Provider) error { } // GetProviders returns all providers -const query_GetProviders = `SELECT id, enabled, priority, name, host, port, ssl, username, password, max_conns, max_art_size, created_at FROM providers order by priority ASC` +const query_GetProviders = `SELECT id, enabled, priority, name, host, port, ssl, username, password, max_conns, max_art_size, posting, created_at, proxy_enabled, proxy_type, proxy_host, proxy_port, proxy_username, proxy_password FROM providers order by priority ASC` func (db *Database) GetProviders() ([]*models.Provider, error) { rows, err := retryableQuery(db.mainDB, query_GetProviders) @@ -101,7 +106,7 @@ func (db *Database) GetProviders() ([]*models.Provider, error) { var out []*models.Provider for rows.Next() { var p models.Provider - if err := rows.Scan(&p.ID, &p.Enabled, &p.Priority, &p.Name, &p.Host, &p.Port, &p.SSL, &p.Username, &p.Password, &p.MaxConns, &p.MaxArtSize, &p.CreatedAt); err != nil { + if err := rows.Scan(&p.ID, &p.Enabled, &p.Priority, &p.Name, &p.Host, &p.Port, &p.SSL, &p.Username, &p.Password, &p.MaxConns, &p.MaxArtSize, &p.Posting, &p.CreatedAt, &p.ProxyEnabled, &p.ProxyType, &p.ProxyHost, &p.ProxyPort, &p.ProxyUsername, &p.ProxyPassword); err != nil { return nil, err } out = append(out, &p) @@ -196,6 +201,30 @@ func (db *Database) MainDBGetNewsgroup(newsgroup string) (*models.Newsgroup, err return &g, nil } +// MainDBGetNewsgroupByID returns a newsgroup information from MainDB by ID +const query_MainDBGetNewsgroupByID = `SELECT id, name, description, last_article, message_count, active, expiry_days, max_articles, max_art_size, high_water, low_water, status, hierarchy, created_at FROM newsgroups WHERE id = ?` + +func (db *Database) MainDBGetNewsgroupByID(id int64) (*models.Newsgroup, error) { + rows, err := retryableQuery(db.mainDB, query_MainDBGetNewsgroupByID, id) + if err != nil { + log.Printf("MainDBGetNewsgroupByID: Failed to query newsgroup with ID %d: %v", id, err) + return nil, err + } + defer rows.Close() + var g models.Newsgroup + found := false + for rows.Next() { + if err := rows.Scan(&g.ID, &g.Name, &g.Description, &g.LastArticle, &g.MessageCount, &g.Active, &g.ExpiryDays, &g.MaxArticles, &g.MaxArtSize, &g.HighWater, &g.LowWater, &g.Status, &g.Hierarchy, &g.CreatedAt); err != nil { + return nil, err + } + found = true + } + if !found { + return nil, sql.ErrNoRows + } + return &g, nil +} + // UpdateNewsgroup updates an existing newsgroup const query_UpdateNewsgroup = `UPDATE newsgroups SET description = ?, last_article = ?, message_count = ?, active = ?, expiry_days = ?, max_articles = ?, high_water = ?, low_water = ?, status = ?, hierarchy = ? WHERE name = ?` @@ -467,16 +496,20 @@ func (db *Database) GetLastArticleDate(groupDBs *GroupDBs) (*time.Time, error) { return &lastDate, nil } -const query_GetAllArticles = `SELECT article_num, message_id, subject, from_header, date_sent, date_string, "references", bytes, lines, reply_count, path, headers_json, body_text, imported_at FROM articles ORDER BY article_num ASC` +const query_GetArticlesBatch = `SELECT article_num, message_id, subject, from_header, date_sent, date_string, "references", bytes, lines, reply_count, path, headers_json, body_text, imported_at FROM articles ORDER BY article_num ASC LIMIT ? OFFSET ?` -func (db *Database) GetAllArticles(groupDBs *GroupDBs) ([]*models.Article, error) { - log.Printf("GetArticles: group '%s' fetching articles", groupDBs.Newsgroup) +// GetArticlesBatch retrieves articles from a group database in batches for memory efficiency +func (db *Database) GetArticlesBatch(groupDBs *GroupDBs, limit, offset int) ([]*models.Article, error) { + if limit <= 0 { + limit = 100 // Default batch size + } - rows, err := retryableQuery(groupDBs.DB, query_GetAllArticles) + rows, err := retryableQuery(groupDBs.DB, query_GetArticlesBatch, limit, offset) if err != nil { return nil, err } defer rows.Close() + var out []*models.Article for rows.Next() { var a models.Article @@ -1153,7 +1186,7 @@ func (db *Database) GetSectionGroupsByName(newsgroupName string) ([]*models.Sect return out, nil } -const query_GetProviderByName = `SELECT id, name, grp, host, port, ssl, username, password, max_conns, enabled, priority, max_art_size +const query_GetProviderByName = `SELECT id, name, grp, host, port, ssl, username, password, max_conns, enabled, priority, max_art_size, posting, proxy_enabled, proxy_type, proxy_host, proxy_port, proxy_username, proxy_password FROM providers WHERE name = ? ORDER by id ASC LIMIT 1` func (db *Database) GetProviderByName(name string) (*models.Provider, error) { @@ -1161,7 +1194,7 @@ func (db *Database) GetProviderByName(name string) (*models.Provider, error) { var provider models.Provider err := row.Scan(&provider.ID, &provider.Name, &provider.Grp, &provider.Host, &provider.Port, &provider.SSL, &provider.Username, &provider.Password, &provider.MaxConns, &provider.Enabled, &provider.Priority, - &provider.MaxArtSize) + &provider.MaxArtSize, &provider.Posting, &provider.ProxyEnabled, &provider.ProxyType, &provider.ProxyHost, &provider.ProxyPort, &provider.ProxyUsername, &provider.ProxyPassword) if err == sql.ErrNoRows { return nil, nil // Provider not found } else if err != nil { @@ -1171,15 +1204,15 @@ func (db *Database) GetProviderByName(name string) (*models.Provider, error) { return &provider, nil } -const query_GetProviderByID = `SELECT id, name, grp, host, port, ssl, username, password, max_conns, enabled, priority, max_art_size +const query_GetProviderByID = `SELECT id, name, grp, host, port, ssl, username, password, max_conns, enabled, priority, max_art_size, posting, proxy_enabled, proxy_type, proxy_host, proxy_port, proxy_username, proxy_password FROM providers WHERE id = ? LIMIT 1` -func (db *Database) GetProviderByID(id int) (*models.Provider, error) { +func (db *Database) GetProviderByID(id int64) (*models.Provider, error) { row := db.mainDB.QueryRow(query_GetProviderByID, id) var provider models.Provider err := row.Scan(&provider.ID, &provider.Name, &provider.Grp, &provider.Host, &provider.Port, &provider.SSL, &provider.Username, &provider.Password, &provider.MaxConns, &provider.Enabled, &provider.Priority, - &provider.MaxArtSize) + &provider.MaxArtSize, &provider.Posting, &provider.ProxyEnabled, &provider.ProxyType, &provider.ProxyHost, &provider.ProxyPort, &provider.ProxyUsername, &provider.ProxyPassword) if err == sql.ErrNoRows { return nil, nil // Provider not found } else if err != nil { diff --git a/internal/models/models.go b/internal/models/models.go index 0c6063f1..152401dd 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -12,7 +12,7 @@ import ( // Hierarchy represents a Usenet hierarchy (e.g., comp, alt, rec) type Hierarchy struct { - ID int `json:"id" db:"id"` + ID int64 `json:"id" db:"id"` Name string `json:"name" db:"name"` Description string `json:"description" db:"description"` GroupCount int `json:"group_count" db:"group_count"` @@ -22,24 +22,32 @@ type Hierarchy struct { // Provider represents an NNTP provider configuration type Provider struct { - ID int `json:"id" db:"id"` - Enabled bool `json:"enabled" db:"enabled"` // Whether this provider is enabled - Priority int `json:"priority" db:"priority"` // Priority for load balancing - Grp string `json:"grp" db:"grp"` - Name string `json:"name" db:"name"` - Host string `json:"host" db:"host"` - Port int `json:"port" db:"port"` - SSL bool `json:"ssl" db:"ssl"` - Username string `json:"username" db:"username"` - Password string `json:"password" db:"password"` - MaxConns int `json:"max_conns" db:"max_conns"` // Maximum concurrent connections - MaxArtSize int `json:"max_art_size" db:"max_art_size"` // Maximum article size in bytes - CreatedAt time.Time `json:"created_at" db:"created_at"` + ID int64 `json:"id" db:"id"` + Enabled bool `json:"enabled" db:"enabled"` // Whether this provider is enabled + Priority int `json:"priority" db:"priority"` // Priority for load balancing + Grp string `json:"grp" db:"grp"` + Name string `json:"name" db:"name"` + Host string `json:"host" db:"host"` + Port int `json:"port" db:"port"` + SSL bool `json:"ssl" db:"ssl"` + Username string `json:"username" db:"username"` + Password string `json:"password" db:"password"` + MaxConns int `json:"max_conns" db:"max_conns"` // Maximum concurrent connections + MaxArtSize int `json:"max_art_size" db:"max_art_size"` // Maximum article size in bytes + Posting bool `json:"posting" db:"posting"` // Whether posting is enabled for this provider + // Proxy configuration fields + ProxyEnabled bool `json:"proxy_enabled" db:"proxy_enabled"` // Whether to use proxy for this provider + ProxyType string `json:"proxy_type" db:"proxy_type"` // Proxy type: socks4, socks5 + ProxyHost string `json:"proxy_host" db:"proxy_host"` // Proxy server hostname/IP + ProxyPort int `json:"proxy_port" db:"proxy_port"` // Proxy server port + ProxyUsername string `json:"proxy_username" db:"proxy_username"` // Proxy authentication username + ProxyPassword string `json:"proxy_password" db:"proxy_password"` // Proxy authentication password + CreatedAt time.Time `json:"created_at" db:"created_at"` } // Newsgroup represents a subscribed newsgroup type Newsgroup struct { - ID int `json:"id" db:"id"` + ID int64 `json:"id" db:"id"` Name string `json:"name" db:"name"` Active bool `json:"active" db:"active"` Description string `json:"description" db:"description"` diff --git a/internal/models/postqueue.go b/internal/models/postqueue.go new file mode 100644 index 00000000..09ef08d9 --- /dev/null +++ b/internal/models/postqueue.go @@ -0,0 +1,6 @@ +package models + +// PostQueueChannel is the global channel for articles posted from web interface +// This channel is used to pass articles from the web interface to the processor +// for background processing and insertion into the NNTP system +var PostQueueChannel = make(chan *Article, 100) diff --git a/internal/nntp/nntp-backend-pool.go b/internal/nntp/nntp-backend-pool.go index d7fbed2b..712044b3 100644 --- a/internal/nntp/nntp-backend-pool.go +++ b/internal/nntp/nntp-backend-pool.go @@ -14,7 +14,7 @@ import ( // Pool manages a pool of NNTP client connections type Pool struct { mux sync.RWMutex - Backend *BackendConfig + Backend *BackendConfig // links to internal/nntp/nntp-client.go:68 connections chan *BackendConn maxConns int activeConns int @@ -68,7 +68,7 @@ func (p *Pool) XOver(group string, start, end int64, enforceLimit bool) ([]Overv return result, nil } -func (p *Pool) XHdr(group string, header string, start, end int64) ([]HeaderLine, error) { +func (p *Pool) XHdr(group string, header string, start, end int64) ([]*HeaderLine, error) { // Get a connection from the pool client, err := p.Get() if err != nil { @@ -90,18 +90,18 @@ func (p *Pool) XHdr(group string, header string, start, end int64) ([]HeaderLine // XHdrStreamed performs XHDR command and streams results through a channel // The channel will be closed when all results are sent or an error occurs // NOTE: This function takes ownership of the connection and will return it to the pool when done -func (p *Pool) XHdrStreamed(group string, header string, start, end int64, resultChan chan<- *HeaderLine) error { +func (p *Pool) XHdrStreamed(group string, header string, start, end int64, xhdrChan chan<- *HeaderLine, shutdownChan <-chan struct{}) error { // Get a connection from the pool client, err := p.Get() if err != nil { - close(resultChan) + close(xhdrChan) return fmt.Errorf("failed to get connection: %w", err) } // Handle connection cleanup in a goroutine so the function can return immediately - go func(client *BackendConn, group string, header string, start, end int64, resultChan chan<- *HeaderLine) { + go func(client *BackendConn, group string, header string, start, end int64, resultChan chan<- *HeaderLine, shutdownChan <-chan struct{}) { // Use the streaming XHdr function on the client - if err := client.XHdrStreamed(group, header, start, end, resultChan); err != nil { + if err := client.XHdrStreamed(group, header, start, end, resultChan, shutdownChan); err != nil { // If there's an error, close the connection instead of returning it err := p.CloseConn(client, true) if err != nil { @@ -110,7 +110,7 @@ func (p *Pool) XHdrStreamed(group string, header string, start, end int64, resul } else { p.Put(client) } - }(client, group, header, start, end, resultChan) + }(client, group, header, start, end, xhdrChan, shutdownChan) return err } @@ -258,7 +258,6 @@ func (p *Pool) Put(client *BackendConn) error { client.CloseFromPoolOnly() } else { log.Printf("[NNTP-POOL] ERROR: Attempted to put nil client back into pool") - p.mux.Unlock() } p.mux.Lock() p.totalClosed++ @@ -274,8 +273,7 @@ func (p *Pool) Put(client *BackendConn) error { case p.connections <- client: return nil default: - log.Printf("[NNTP-POOL ERROR: Pool is full ?! should be fatal! closing connection for %s:%d", p.Backend.Host, p.Backend.Port) - // Pool is full, close the connection + log.Printf("[NNTP-POOL] ERROR: Pool is full or closed. Closing conn for %s:%d", p.Backend.Host, p.Backend.Port) client.CloseFromPoolOnly() p.mux.Lock() p.totalClosed++ @@ -325,10 +323,12 @@ func (p *Pool) ClosePool() error { p.totalClosed++ p.mux.Unlock() } + p.mux.Lock() if p.activeConns > 0 { log.Printf("[NNTP-POOL] WARNING: Pool closed with positive count %d active connections remaining ?!?!", p.activeConns) } p.activeConns = 0 + p.mux.Unlock() return nil } diff --git a/internal/nntp/nntp-client-commands.go b/internal/nntp/nntp-client-commands.go index fcbc10ac..40e6f1cc 100644 --- a/internal/nntp/nntp-client-commands.go +++ b/internal/nntp/nntp-client-commands.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/go-while/go-pugleaf/internal/common" "github.com/go-while/go-pugleaf/internal/models" "github.com/go-while/go-pugleaf/internal/utils" ) @@ -310,8 +311,8 @@ func (c *BackendConn) ListGroupsLimited(maxGroups int) ([]GroupInfo, error) { for { if lineCount >= maxGroups { - c.textConn.Close() // Close connection on limit reached - c = nil + c.conn.Close() // Close connection on limit reached + c.conn = nil log.Printf("Connection reached maximum group limit: %d", maxGroups) break } @@ -492,7 +493,7 @@ func (c *BackendConn) XOver(groupName string, start, end int64, enforceLimit boo // XHdr retrieves specific header field for a range of articles // Automatically limits to max 1000 articles to prevent SQLite overload -func (c *BackendConn) XHdr(groupName, field string, start, end int64) ([]HeaderLine, error) { +func (c *BackendConn) XHdr(groupName, field string, start, end int64) ([]*HeaderLine, error) { c.mu.Lock() if !c.connected { c.mu.Unlock() @@ -540,7 +541,7 @@ func (c *BackendConn) XHdr(groupName, field string, start, end int64) ([]HeaderL } // Parse header lines - var headers = make([]HeaderLine, 0, len(lines)) + var headers = make([]*HeaderLine, 0, len(lines)) for _, line := range lines { header, err := c.parseHeaderLine(line) if err != nil { @@ -554,30 +555,97 @@ func (c *BackendConn) XHdr(groupName, field string, start, end int64) ([]HeaderL var ErrOutOfRange error = fmt.Errorf("end range exceeds group last article number") +func (c *BackendConn) WantShutdown(shutdownChan <-chan struct{}) bool { + select { + case _, ok := <-shutdownChan: + if !ok { + // channel is closed + return true + } + default: + } + return false +} + // XHdrStreamed performs XHDR command and streams results line by line through a channel -func (c *BackendConn) XHdrStreamed(groupName, field string, start, end int64, resultChan chan<- *HeaderLine) error { +// Fetches max 1000 hdrs and starts a new fetch if the channel is less than 10% capacity +func (c *BackendConn) XHdrStreamed(groupName, field string, start, end int64, xhdrChan chan<- *HeaderLine, shutdownChan <-chan struct{}) error { + channelCap := cap(xhdrChan) + lowWaterMark := channelCap / 10 // 10% threshold + if lowWaterMark < 1 { + lowWaterMark = 1 + } + + currentStart := start + var isleep int64 = 10 + for currentStart <= end { + // Check for shutdown signal + if c.WantShutdown(shutdownChan) { + close(xhdrChan) + log.Printf("XHdrStreamed: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } + + // Wait if channel is not empty + for len(xhdrChan) > lowWaterMark { + time.Sleep(time.Duration(isleep) * time.Millisecond) + if c.WantShutdown(shutdownChan) { + close(xhdrChan) + log.Printf("XHdrStreamed: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } + } + + // Calculate batch end (max 1000 articles) + batchEnd := currentStart + 999 // 1000 articles max + if batchEnd > end { + batchEnd = end + } + + // Fetch this batch + startStream := time.Now() + err := c.XHdrStreamedBatch(groupName, field, currentStart, batchEnd, xhdrChan, shutdownChan) + if err != nil { + close(xhdrChan) // Close on error + return fmt.Errorf("XHdrStreamedBatch failed for range %d-%d: %w", currentStart, batchEnd, err) + } + isleep = time.Since(startStream).Milliseconds() / 2 + if isleep < 10 { + isleep = 10 + } + + // Move to next batch + currentStart = batchEnd + 1 + + } + + // Close channel when all batches are complete + close(xhdrChan) + return nil +} + +// XHdrStreamedBatch performs XHDR command and streams results line by line through a channel +func (c *BackendConn) XHdrStreamedBatch(groupName, field string, start, end int64, xhdrChan chan<- *HeaderLine, shutdownChan <-chan struct{}) error { c.mu.Lock() if !c.connected { c.mu.Unlock() - close(resultChan) return fmt.Errorf("not connected") } c.mu.Unlock() groupInfo, code, err := c.SelectGroup(groupName) if err != nil && code != 411 { - close(resultChan) return fmt.Errorf("failed to select group '%s': code=%d err=%w", groupName, code, err) } if end > groupInfo.Last { - close(resultChan) return ErrOutOfRange } c.lastUsed = time.Now() // Limit to 1000 articles maximum to prevent SQLite overload - if end > 0 && (end-start+1) > MaxReadLinesXover { - end = start + MaxReadLinesXover - 1 + const maxFetchLimit = 1000 + if end > 0 && (end-start+1) > maxFetchLimit { + end = start + maxFetchLimit - 1 } //log.Printf("XHdrStreamed group '%s' field '%s' start=%d end=%d", groupName, field, start, end) @@ -588,46 +656,37 @@ func (c *BackendConn) XHdrStreamed(groupName, field string, start, end int64, re id, err = c.textConn.Cmd("XHDR %s %d-%d", field, start, start) } if err != nil { - close(resultChan) return fmt.Errorf("failed to send XHDR command: %w", err) } c.textConn.StartResponse(id) defer c.textConn.EndResponse(id) // Always clean up response state - // Set timeout for initial response - /* - if err := c.conn.SetReadDeadline(time.Now().Add(9 * time.Second)); err != nil { - close(resultChan) - return fmt.Errorf("failed to set read deadline: %w", err) - } - defer func() { - // Clear the deadline when operation completes - if c.conn != nil { - c.conn.SetReadDeadline(time.Time{}) - } - }() - */ + // Check for shutdown before reading initial response + if c.WantShutdown(shutdownChan) { + log.Printf("XHdrStreamed: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } + code, message, err := c.textConn.ReadCodeLine(221) if err != nil { - close(resultChan) return fmt.Errorf("failed to read XHDR response: %w", err) } if code != 221 { - close(resultChan) return fmt.Errorf("XHDR failed: ng: '%s' %d %s", groupName, code, message) } // Read multiline response line by line and send to channel immediately for { - /* - // Set a shorter timeout for each line read (3 seconds per line) - if err := c.conn.SetReadDeadline(time.Now().Add(3 * time.Second)); err != nil { - log.Printf("[ERROR] XHdrStreamed failed to set line deadline ng: '%s' err='%v'", groupName, err) - break - } - */ + // Check for shutdown signal between reads + if c.WantShutdown(shutdownChan) { + c.conn.Close() // Close connection on shutdown + c.conn = nil + log.Printf("XHdrStreamed: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } + line, err := c.textConn.ReadLine() if err != nil { log.Printf("[ERROR] XHdrStreamed read error ng: '%s' err='%v'", groupName, err) @@ -647,12 +706,17 @@ func (c *BackendConn) XHdrStreamed(groupName, field string, start, end int64, re continue // Skip malformed lines } - // Send through channel - resultChan <- &header + if c.WantShutdown(shutdownChan) { + c.conn.Close() // Close connection on shutdown + c.conn = nil + log.Printf("XHdrStreamed: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } + + xhdrChan <- header } - // Close channel - close(resultChan) + // Don't close channel here - let the main function handle it return nil } @@ -756,8 +820,8 @@ func (c *BackendConn) readMultilineResponse(src string) ([]string, error) { } for { if lineCount >= maxReadLines { - c.textConn.Close() // Close connection on too many lines - c = nil + c.conn.Close() // Close connection on limit reached + c.conn = nil return nil, fmt.Errorf("too many lines in response (limit: %d)", maxReadLines) } @@ -949,16 +1013,382 @@ func (c *BackendConn) parseOverviewLine(line string) (OverviewLine, error) { // parseHeaderLine parses a single XHDR response line // Format: articlenumheader-value -func (c *BackendConn) parseHeaderLine(line string) (HeaderLine, error) { +func (c *BackendConn) parseHeaderLine(line string) (*HeaderLine, error) { parts := strings.SplitN(line, " ", 2) if len(parts) < 2 { - return HeaderLine{}, fmt.Errorf("malformed XHDR line: %s", line) + return nil, fmt.Errorf("malformed XHDR line: %s", line) } - articleNum, _ := strconv.ParseInt(parts[0], 10, 64) + articleNum, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + log.Printf("Invalid article number in XHDR line: %q", parts[0]) + return nil, fmt.Errorf("invalid article number in XHDR line: %q", parts[0]) + } - return HeaderLine{ + return &HeaderLine{ ArticleNum: articleNum, Value: parts[1], }, nil } + +// CheckResponse represents a response to CHECK command +type CheckResponse struct { + MessageID *string + Wanted bool + Code int +} + +// CheckMultiple sends a CHECK command for multiple message IDs and returns responses +func (c *BackendConn) CheckMultiple(messageIDs []*string) ([]CheckResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.connected { + return nil, fmt.Errorf("not connected") + } + + if len(messageIDs) == 0 { + return nil, fmt.Errorf("no message IDs provided") + } + + c.lastUsed = time.Now() + + // Send individual CHECK commands for each message ID (pipelining) + commandIds := make([]uint, len(messageIDs)) + for i, msgID := range messageIDs { + id, err := c.textConn.Cmd("CHECK %s", *msgID) + if err != nil { + return nil, fmt.Errorf("failed to send CHECK command for %s: %w", *msgID, err) + } + commandIds[i] = id + } + + // Read responses for each CHECK command + responses := make([]CheckResponse, 0, len(messageIDs)) + var outoforder []CheckResponse + for i, msgID := range messageIDs { + id := commandIds[i] + c.textConn.StartResponse(id) + + // Read response for this CHECK command + code, line, err := c.textConn.ReadCodeLine(238) + c.textConn.EndResponse(id) + + if code == 0 && err != nil { + log.Printf("Failed to read CHECK response for %s: %v", *msgID, err) + continue + } + + // Parse response line + // Format: code [message] + // 238 - article wanted + // 431 - article not wanted + // 438 - article not wanted (already have it) + // ReadCodeLine returns: code=238, message=" article wanted" + parts := strings.Fields(line) + if len(parts) < 1 { + log.Printf("Malformed CHECK response: %s", line) + continue + } + if parts[0] != *msgID { + log.Printf("Mismatched CHECK response: expected %s, got %s", *msgID, parts[0]) + outoforder = append(outoforder, CheckResponse{ + MessageID: &parts[0], + Code: code, + Wanted: code == 238, // 238 means article wanted + }) + continue + } + + response := CheckResponse{ + MessageID: msgID, // First part is the message ID + Code: code, + Wanted: code == 238, // 238 means article wanted + } + + responses = append(responses, response) + } + + for _, resp := range outoforder { + for _, msgID := range messageIDs { + if *resp.MessageID == *msgID { + responses = append(responses, CheckResponse{ + MessageID: msgID, // First part is the message ID + Code: resp.Code, + Wanted: resp.Wanted, + }) + } + } + } + + // Return all responses + + return responses, nil +} + +// TakeThisArticle sends an article via TAKETHIS command +func (c *BackendConn) TakeThisArticle(article *models.Article, nntphostname *string) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.connected { + return 0, fmt.Errorf("not connected") + } + + // Prepare article for transfer + headers, err := common.ReconstructHeaders(article, true, nntphostname) + if err != nil { + return 0, fmt.Errorf("failed to reconstruct headers: %v", err) + } + + c.lastUsed = time.Now() + + // Send TAKETHIS command + id, err := c.textConn.Cmd("TAKETHIS %s", article.MessageID) + if err != nil { + return 0, fmt.Errorf("failed to send TAKETHIS command: %w", err) + } + + // Send headers + for _, headerLine := range headers { + if _, err := c.writer.WriteString(headerLine + CRLF); err != nil { + return 0, fmt.Errorf("failed to write header: %w", err) + } + } + + // Send empty line between headers and body + if _, err := c.writer.WriteString(CRLF); err != nil { + return 0, fmt.Errorf("failed to write header/body separator: %w", err) + } + + // Send body with proper dot-stuffing + // Split body preserving line endings + bodyLines := strings.Split(article.BodyText, "\n") + for i, line := range bodyLines { + // Skip empty last element from trailing \n + if i == len(bodyLines)-1 && line == "" { + break + } + + // Remove trailing \r if present (will add CRLF) + line = strings.TrimSuffix(line, "\r") + + // Dot-stuff lines that start with a dot (RFC 977) + if strings.HasPrefix(line, ".") { + line = "." + line + } + + if _, err := c.writer.WriteString(line + CRLF); err != nil { + return 0, fmt.Errorf("failed to write body line: %w", err) + } + } + + // Send termination line (single dot) + if _, err := c.writer.WriteString(DOT + CRLF); err != nil { + return 0, fmt.Errorf("failed to send article terminator: %w", err) + } + + // Flush the writer to ensure all data is sent + if err := c.writer.Flush(); err != nil { + return 0, fmt.Errorf("failed to flush article data: %w", err) + } + + // Read TAKETHIS response + c.textConn.StartResponse(id) + defer c.textConn.EndResponse(id) + + code, _, err := c.textConn.ReadCodeLine(239) // -1 means any code is acceptable + if code == 0 && err != nil { + return 0, fmt.Errorf("failed to read TAKETHIS response: %w", err) + } + + // Parse response + // Format: code [message] + // 239 - article transferred successfully + // 439 - article transfer failed + + return code, nil +} + +// SendTakeThisArticleStreaming sends TAKETHIS command and article content without waiting for response +// Returns command ID for later response reading - used for streaming mode +func (c *BackendConn) SendTakeThisArticleStreaming(article *models.Article, nntphostname *string) (uint, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.connected { + return 0, fmt.Errorf("not connected") + } + + // Prepare article for transfer + headers, err := common.ReconstructHeaders(article, true, nntphostname) + if err != nil { + return 0, fmt.Errorf("failed to reconstruct headers: %v", err) + } + + c.lastUsed = time.Now() + + // Send TAKETHIS command + id, err := c.textConn.Cmd("TAKETHIS %s", article.MessageID) + if err != nil { + return 0, fmt.Errorf("failed to send TAKETHIS command: %w", err) + } + + // Send headers + for _, headerLine := range headers { + if _, err := c.writer.WriteString(headerLine + CRLF); err != nil { + return 0, fmt.Errorf("failed to write header: %w", err) + } + } + + // Send empty line between headers and body + if _, err := c.writer.WriteString(CRLF); err != nil { + return 0, fmt.Errorf("failed to write header/body separator: %w", err) + } + + // Send body with proper dot-stuffing + // Split body preserving line endings + bodyLines := strings.Split(article.BodyText, "\n") + for i, line := range bodyLines { + // Skip empty last element from trailing \n + if i == len(bodyLines)-1 && line == "" { + break + } + + // Remove trailing \r if present (will add CRLF) + line = strings.TrimSuffix(line, "\r") + + // Dot-stuff lines that start with a dot (RFC 977) + if strings.HasPrefix(line, ".") { + line = "." + line + } + + if _, err := c.writer.WriteString(line + CRLF); err != nil { + return 0, fmt.Errorf("failed to write body line: %w", err) + } + } + + // Send termination line (single dot) + if _, err := c.writer.WriteString(DOT + CRLF); err != nil { + return 0, fmt.Errorf("failed to send article terminator: %w", err) + } + + // Flush the writer to ensure all data is sent + if err := c.writer.Flush(); err != nil { + return 0, fmt.Errorf("failed to flush article data: %w", err) + } + + // Return command ID without reading response (streaming mode) + return id, nil +} + +// ReadTakeThisResponseStreaming reads a TAKETHIS response using the command ID +// Used in streaming mode after all articles have been sent +func (c *BackendConn) ReadTakeThisResponseStreaming(id uint) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Read TAKETHIS response + c.textConn.StartResponse(id) + defer c.textConn.EndResponse(id) + + code, _, err := c.textConn.ReadCodeLine(239) + if code == 0 && err != nil { + return 0, fmt.Errorf("failed to read TAKETHIS response: %w", err) + } + + // Parse response + // Format: code [message] + // 239 - article transferred successfully + // 439 - article transfer failed + return code, nil +} + +// PostArticle posts an article using the POST command +func (c *BackendConn) PostArticle(article *models.Article) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.connected { + return 0, fmt.Errorf("not connected") + } + // Prepare article for posting + headers, err := common.ReconstructHeaders(article, false, nil) + if err != nil { + return 0, fmt.Errorf("failed to reconstruct headers: %v", err) + } + c.lastUsed = time.Now() + + // Send POST command + id, err := c.textConn.Cmd("POST") + if err != nil { + return 0, fmt.Errorf("failed to send POST command: %w", err) + } + + c.textConn.StartResponse(id) + defer c.textConn.EndResponse(id) + + // Read response to POST command + code, line, err := c.textConn.ReadCodeLine(340) + if err != nil { + return code, fmt.Errorf("POST command failed: %s", line) + } + if code != 340 { + return code, fmt.Errorf("POST command rejected (code %d): %s", code, line) + } + + // Send headers using writer (not DotWriter) + for _, headerLine := range headers { + if _, err := c.writer.WriteString(headerLine + CRLF); err != nil { + return 0, fmt.Errorf("failed to write header: %w", err) + } + } + + // Send empty line between headers and body + if _, err := c.writer.WriteString(CRLF); err != nil { + return 0, fmt.Errorf("failed to write header/body separator: %w", err) + } + + // Send body with proper dot-stuffing (like TakeThisArticle) + // Split body preserving line endings + bodyLines := strings.Split(article.BodyText, "\n") + for i, line := range bodyLines { + // Skip empty last element from trailing \n + if i == len(bodyLines)-1 && line == "" { + break + } + + // Remove trailing \r if present (will add CRLF) + line = strings.TrimSuffix(line, "\r") + + // Dot-stuff lines that start with a dot (RFC 977) + if strings.HasPrefix(line, ".") { + line = "." + line + } + + if _, err := c.writer.WriteString(line + CRLF); err != nil { + return 0, fmt.Errorf("failed to write body line: %w", err) + } + } + + // Send termination line (single dot) + if _, err := c.writer.WriteString(DOT + CRLF); err != nil { + return 0, fmt.Errorf("failed to send article terminator: %w", err) + } + + // Flush the writer to ensure all data is sent + if err := c.writer.Flush(); err != nil { + return 0, fmt.Errorf("failed to flush article data: %w", err) + } + + // Read final response + code, _, err = c.textConn.ReadCodeLine(240) + if err != nil { + return code, fmt.Errorf("failed to read POST response: %w", err) + } + + // Parse response codes + // 240 - article posted successfully + // 441 - posting failed + return code, nil +} diff --git a/internal/nntp/nntp-client.go b/internal/nntp/nntp-client.go index 7af58510..2aa95194 100644 --- a/internal/nntp/nntp-client.go +++ b/internal/nntp/nntp-client.go @@ -76,7 +76,14 @@ type BackendConfig struct { //WriteTimeout time.Duration // timeout for writing to the connection MaxConns int // maximum number of connections to this backend Provider *config.Provider // link to provider config - Mux sync.Mutex + // Proxy configuration fields + ProxyEnabled bool // whether to use proxy for connections + ProxyType string // proxy type: socks4, socks5 + ProxyHost string // proxy server hostname/IP + ProxyPort int // proxy server port + ProxyUsername string // proxy authentication username + ProxyPassword string // proxy authentication password + Mux sync.Mutex } // Article represents an NNTP article @@ -91,11 +98,13 @@ type Article struct { // GroupInfo represents newsgroup information type GroupInfo struct { - Name string - Count int64 - First int64 - Last int64 - PostingOK bool + Name string + Count int64 + First int64 + Last int64 + FetchStart int64 + FetchEnd int64 + PostingOK bool } // OverviewLine represents a line from XOVER command @@ -136,9 +145,19 @@ func (c *BackendConn) Connect() error { if c.connected { return nil } + + // Check if this is a .onion address and automatically enable Tor if not already configured + if IsOnionAddress(c.Backend.Host) && !c.Backend.ProxyEnabled { + log.Printf("[NNTP-PROXY] Detected .onion address %s but TOR proxy is not configured!", c.Backend.Host) + return fmt.Errorf("error in Connect: TOR proxy is not configured: %s", c.Backend.Provider.Name) + } + // Build server address serverAddr := net.JoinHostPort(c.Backend.Host, fmt.Sprintf("%d", c.Backend.Port)) + // Create proxy dialer + proxyDialer := NewProxyDialer(c.Backend) + // Set connection timeout var conn net.Conn var err error @@ -148,17 +167,29 @@ func (c *BackendConn) Connect() error { ServerName: c.Backend.Host, MinVersion: tls.VersionTLS12, } - conn, err = tls.DialWithDialer(&net.Dialer{ - Timeout: c.Backend.ConnectTimeout, - }, "tcp", serverAddr, tlsConfig) + + // Use proxy-aware TLS dialer + conn, err = proxyDialer.DialTLS("tcp", serverAddr, tlsConfig) } else { - conn, err = net.DialTimeout("tcp", serverAddr, c.Backend.ConnectTimeout) + // Use proxy-aware dialer + conn, err = proxyDialer.Dial("tcp", serverAddr) } if err != nil { + // Log proxy information if proxy was attempted + if c.Backend.ProxyEnabled { + log.Printf("[NNTP-PROXY] Failed to connect to %s via %s proxy %s:%d: %v", + serverAddr, c.Backend.ProxyType, c.Backend.ProxyHost, c.Backend.ProxyPort, err) + } return fmt.Errorf("failed to connect to %s: %w", serverAddr, err) } + // Log successful proxy connection + if c.Backend.ProxyEnabled { + log.Printf("[NNTP-PROXY] Successfully connected to %s via %s proxy %s:%d", + serverAddr, c.Backend.ProxyType, c.Backend.ProxyHost, c.Backend.ProxyPort) + } + c.conn = conn c.textConn = textproto.NewConn(conn) c.writer = bufio.NewWriter(conn) diff --git a/internal/nntp/proxy.go b/internal/nntp/proxy.go new file mode 100644 index 00000000..1c30a359 --- /dev/null +++ b/internal/nntp/proxy.go @@ -0,0 +1,121 @@ +package nntp + +import ( + "crypto/tls" + "fmt" + "net" + "strings" + + "golang.org/x/net/proxy" +) + +// ProxyType represents the type of proxy connection +type ProxyType string + +const ( + ProxyTypeSOCKS4 ProxyType = "socks4" + ProxyTypeSOCKS5 ProxyType = "socks5" +) + +// ProxyDialer provides proxy-aware dialing capabilities for NNTP connections +type ProxyDialer struct { + config *BackendConfig +} + +// NewProxyDialer creates a new proxy dialer based on the backend configuration +func NewProxyDialer(config *BackendConfig) *ProxyDialer { + return &ProxyDialer{ + config: config, + } +} + +// Dial establishes a connection to the target host, optionally through a proxy +func (pd *ProxyDialer) Dial(network, address string) (net.Conn, error) { + // If proxy is not enabled, use direct connection + if !pd.config.ProxyEnabled { + return pd.dialDirect(network, address) + } + + switch ProxyType(pd.config.ProxyType) { + case ProxyTypeSOCKS4: + return pd.dialSOCKS4(network, address) + case ProxyTypeSOCKS5: + return pd.dialSOCKS5(network, address) + default: + return nil, fmt.Errorf("unsupported proxy type: %s", pd.config.ProxyType) + } +} + +// DialTLS establishes a TLS connection to the target host, optionally through a proxy +func (pd *ProxyDialer) DialTLS(network, address string, tlsConfig *tls.Config) (net.Conn, error) { + // First establish the connection (direct or through proxy) + // pd.Dial() handles all the proxy logic internally + conn, err := pd.Dial(network, address) + if err != nil { + return nil, err + } + + // Now wrap the connection (whether direct or proxied) with TLS + tlsConn := tls.Client(conn, tlsConfig) + if err := tlsConn.Handshake(); err != nil { + conn.Close() + if pd.config.ProxyEnabled { + return nil, fmt.Errorf("TLS handshake failed through %s proxy: %w", pd.config.ProxyType, err) + } + return nil, fmt.Errorf("TLS handshake failed: %w", err) + } + + return tlsConn, nil +} + +// dialDirect establishes a direct connection without proxy +func (pd *ProxyDialer) dialDirect(network, address string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: pd.config.ConnectTimeout, + } + return dialer.Dial(network, address) +} + +// dialSOCKS4 establishes a connection through a SOCKS4 proxy +// Note: golang.org/x/net/proxy doesn't have native SOCKS4 support, so we use SOCKS5 without auth +func (pd *ProxyDialer) dialSOCKS4(network, address string) (net.Conn, error) { + proxyAddr := net.JoinHostPort(pd.config.ProxyHost, fmt.Sprintf("%d", pd.config.ProxyPort)) + + // SOCKS4 doesn't support authentication, so we use SOCKS5 without auth + // Most SOCKS4 proxies also support SOCKS5 without authentication (said AI) + dialer, err := proxy.SOCKS5("tcp", proxyAddr, nil, &net.Dialer{ + Timeout: pd.config.ConnectTimeout, + }) + if err != nil { + return nil, fmt.Errorf("failed to create SOCKS4/5 proxy dialer: %w", err) + } + + return dialer.Dial(network, address) +} + +// dialSOCKS5 establishes a connection through a SOCKS5 proxy (including Tor) +func (pd *ProxyDialer) dialSOCKS5(network, address string) (net.Conn, error) { + proxyAddr := net.JoinHostPort(pd.config.ProxyHost, fmt.Sprintf("%d", pd.config.ProxyPort)) + + var auth *proxy.Auth + if pd.config.ProxyUsername != "" { + auth = &proxy.Auth{ + User: pd.config.ProxyUsername, + Password: pd.config.ProxyPassword, + } + } + + dialer, err := proxy.SOCKS5("tcp", proxyAddr, auth, &net.Dialer{ + Timeout: pd.config.ConnectTimeout, + }) + if err != nil { + return nil, fmt.Errorf("failed to create SOCKS5 proxy dialer: %w", err) + } + + return dialer.Dial(network, address) +} + +// IsOnionAddress checks if the given host is a Tor .onion address +func IsOnionAddress(host string) bool { + return strings.HasSuffix(strings.ToLower(host), ".onion") +} diff --git a/internal/postmgr/postmgr.go b/internal/postmgr/postmgr.go new file mode 100644 index 00000000..aae798e1 --- /dev/null +++ b/internal/postmgr/postmgr.go @@ -0,0 +1,197 @@ +// Package postmgr provides article posting management for go-pugleaf +package postmgr + +import ( + "fmt" + "log" + "sync" + "time" + + "github.com/go-while/go-pugleaf/internal/database" + "github.com/go-while/go-pugleaf/internal/models" + "github.com/go-while/go-pugleaf/internal/nntp" +) + +// PosterManager manages the posting of articles to multiple providers +type PosterManager struct { + db *database.Database + pools []*nntp.Pool + stopCh chan struct{} + wg sync.WaitGroup +} + +// NewPosterManager creates a new poster manager +func NewPosterManager(db *database.Database, pools []*nntp.Pool) *PosterManager { + return &PosterManager{ + db: db, + pools: pools, + stopCh: make(chan struct{}), + } +} + +// Run starts the poster manager +func (pm *PosterManager) Run(limit int, shutdownChan <-chan struct{}, wg *sync.WaitGroup) { + log.Printf("PosterManager: Starting with %d pools", len(pm.pools)) + defer wg.Done() + for { + select { + case <-shutdownChan: + log.Printf("PosterManager: Received shutdown signal") + close(pm.stopCh) + pm.wg.Wait() + return + case <-pm.stopCh: + pm.wg.Wait() + return + default: + // Check for pending posts + done, err := pm.ProcessPendingPosts(limit) + if err != nil { + log.Printf("PosterManager: Error processing pending posts: %v", err) + } + if done == limit { + time.Sleep(1 * time.Second) + } else { + time.Sleep(10 * time.Second) + } + } + } +} + +// Stop stops the poster manager +func (pm *PosterManager) Stop() { + close(pm.stopCh) + pm.wg.Wait() +} + +// ProcessPendingPosts processes all pending posts in the queue +func (pm *PosterManager) ProcessPendingPosts(limit int) (int, error) { + // Get pending posts from database + entries, err := pm.db.GetPendingPostQueueEntries(limit) + if err != nil { + return 0, fmt.Errorf("failed to get pending posts: %v", err) + } + + if len(entries) == 0 { + log.Printf("No pending posts found") + return 0, nil + } + + log.Printf("Found %d pending posts to process", len(entries)) + + var failedEntryIDs []int64 + + // Process each entry + for _, entry := range entries { + select { + case <-pm.stopCh: + log.Printf("PosterManager: Stopping due to shutdown signal") + // Reset processing state for any remaining entries + if len(failedEntryIDs) > 0 { + pm.db.ResetPostQueueProcessing(failedEntryIDs) + } + return 0, nil + default: + if err := pm.processEntry(entry); err != nil { + log.Printf("Failed to process entry %d (message: %s): %v", entry.ID, entry.MessageID, err) + failedEntryIDs = append(failedEntryIDs, entry.ID) + } + } + } + + // Reset processing state for failed entries + if len(failedEntryIDs) > 0 { + if err := pm.db.ResetPostQueueProcessing(failedEntryIDs); err != nil { + log.Printf("Failed to reset processing state for failed entries: %v", err) + } + } + + return len(entries), nil +} + +// processEntry processes a single post queue entry +func (pm *PosterManager) processEntry(entry database.PostQueueEntry) error { + log.Printf("Processing post queue entry %d (message: %s)", entry.ID, entry.MessageID) + + // Get the newsgroup by ID + newsgroup, err := pm.db.MainDBGetNewsgroupByID(entry.NewsgroupID) + if err != nil { + return fmt.Errorf("failed to get newsgroup with ID %d: %v", entry.NewsgroupID, err) + } + + if !newsgroup.Active { + log.Printf("Newsgroup %s is not active, skipping", newsgroup.Name) + return pm.db.MarkPostQueueAsPostedToRemote(entry.ID) + } + + // Get the article from the local database + article, err := pm.getArticleByMessageID(entry.MessageID, newsgroup.Name) + if err != nil { + return fmt.Errorf("failed to get article %s: %v", entry.MessageID, err) + } + + // Post to all available providers + successCount := 0 + for _, pool := range pm.pools { + if err := pm.postToProvider(article, pool); err != nil { + log.Printf("Failed to post article %s to provider %s: %v", + entry.MessageID, pool.Backend.Provider.Name, err) + } else { + successCount++ + log.Printf("Successfully posted article %s to provider %s", + entry.MessageID, pool.Backend.Provider.Name) + } + } + + if successCount > 0 { + log.Printf("Successfully posted article %s to %d/%d providers", + entry.MessageID, successCount, len(pm.pools)) + return pm.db.MarkPostQueueAsPostedToRemote(entry.ID) + } + + return fmt.Errorf("failed to post article %s to any provider", entry.MessageID) +} + +// getArticleByMessageID retrieves an article from the local database +func (pm *PosterManager) getArticleByMessageID(messageID, newsgroup string) (*models.Article, error) { + // Get group database connection + groupDBs, err := pm.db.GetGroupDBs(newsgroup) + if err != nil { + return nil, err + } + defer groupDBs.Return(pm.db) + + // Get article by message ID using the database method (not groupDBs method) + article, err := pm.db.GetArticleByMessageID(groupDBs, messageID) + if err != nil { + return nil, err + } + + return article, nil +} + +// postToProvider posts an article to a specific provider +func (pm *PosterManager) postToProvider(article *models.Article, pool *nntp.Pool) error { + // Get connection from pool using the correct method name + conn, err := pool.Get() + if err != nil { + return fmt.Errorf("failed to get connection: %v", err) + } + defer pool.Put(conn) + + // Use POST for posting articles (standard NNTP posting) + responseCode, err := conn.PostArticle(article) + if err != nil { + return fmt.Errorf("failed to post article: %v", err) + } + + // Check response code + switch responseCode { + case 240: // Article posted successfully + return nil + case 441: // Posting failed + return fmt.Errorf("article posting failed (code 441)") + default: + return fmt.Errorf("unexpected response code: %d", responseCode) + } +} diff --git a/internal/processor/PostQueue.go b/internal/processor/PostQueue.go new file mode 100644 index 00000000..5db380ff --- /dev/null +++ b/internal/processor/PostQueue.go @@ -0,0 +1,161 @@ +package processor + +import ( + "fmt" + "log" + "strings" + + "github.com/go-while/go-pugleaf/internal/models" +) + +// PostQueueWorker processes articles from the web posting queue +type PostQueueWorker struct { + processor *Processor + stopCh chan struct{} +} + +// NewPostQueueWorker creates a new post queue worker +func (processor *Processor) NewPostQueueWorker() *PostQueueWorker { + return &PostQueueWorker{ + processor: processor, + stopCh: make(chan struct{}), + } +} + +// Start begins processing articles from the post queue +func (w *PostQueueWorker) Start() { + log.Printf("PostQueueWorker: Starting worker") + go w.processLoop() +} + +// Stop gracefully stops the worker +func (w *PostQueueWorker) Stop() { + log.Printf("PostQueueWorker: Stopping worker") + close(w.stopCh) +} + +// processLoop is the main processing loop that reads from the queue +func (w *PostQueueWorker) processLoop() { + for { + select { + case <-w.stopCh: + log.Printf("PostQueueWorker: Worker stopped") + return + + case article := <-models.PostQueueChannel: + if article == nil { + log.Printf("PostQueueWorker: Received nil article, skipping") + continue + } + + log.Printf("PostQueueWorker: Processing article %s", article.MessageID) + err := w.pre_processArticle(article) + if err != nil { + log.Printf("PostQueueWorker: Error processing article %s: %v", article.MessageID, err) + // TODO: Implement retry logic or dead letter queue + } else { + log.Printf("PostQueueWorker: Successfully processed article %s", article.MessageID) + } + } + } +} + +// processArticle processes a single article from the queue +func (w *PostQueueWorker) pre_processArticle(article *models.Article) error { + // Get newsgroups from the article's headers + newsgroupsHeader, exists := article.Headers["newsgroups"] + if !exists || len(newsgroupsHeader) == 0 { + log.Printf("PostQueueWorker: Article %s has no newsgroups header", article.MessageID) + return nil + } + + // Parse newsgroups (comma-separated) + newsgroups := parseNewsgroups(newsgroupsHeader[0]) + if len(newsgroups) == 0 { + log.Printf("PostQueueWorker: Article %s has no valid newsgroups", article.MessageID) + return nil + } + + log.Printf("PostQueueWorker: Processing article %s for newsgroups: %v", article.MessageID, newsgroups) + errs := 0 + // Process the article for each newsgroup + for _, newsgroup := range newsgroups { + err := w.processArticleForNewsgroup(article, newsgroup) + if err != nil { + errs++ + log.Printf("PostQueueWorker: Error processing article %s for newsgroup %s: %v", + article.MessageID, newsgroup, err) + // Continue with other newsgroups even if one fails + } + } + if errs == len(newsgroups) { + return fmt.Errorf("failed to process article %s for all newsgroups", article.MessageID) + } + // Record in post_queue table for tracking + // Get newsgroup ID + newsgroupModel, err := w.processor.DB.GetNewsgroupByName(newsgroups[0]) + if err != nil { + log.Printf("Warning: Failed to get newsgroup %s for post queue recording: %v", newsgroups[0], err) + return fmt.Errorf("failed to get newsgroup %s: %v", newsgroups[0], err) + } + + // Insert into post_queue table + err = w.processor.DB.InsertPostQueueEntry(newsgroupModel.ID, article.MessageID) + if err != nil { + log.Printf("Warning: Failed to insert post_queue entry for newsgroup %s, message_id %s: %v", + newsgroups[0], article.MessageID, err) + } + return nil +} + +// processArticleForNewsgroup processes an article for a specific newsgroup +func (w *PostQueueWorker) processArticleForNewsgroup(article *models.Article, newsgroup string) error { + // Get newsgroup from database + ng, err := w.processor.DB.GetNewsgroupByName(newsgroup) + if err != nil { + return err + } + + if !ng.Active { + log.Printf("PostQueueWorker: Newsgroup %s is not active, skipping", newsgroup) + return nil + } + + // Get group database connection + groupDBs, err := w.processor.DB.GetGroupDBs(newsgroup) + if err != nil { + return err + } + defer groupDBs.Return(w.processor.DB) + + // Use the existing threading function to process the article + // This will handle all the threading logic, database insertion, etc. + //log.Printf("PostQueueWorker: Threading article %s in newsgroup %s", article.MessageID, newsgroup) + + // Process through the threading system using the processor's method + _, err = w.processor.processArticle(article, newsgroup, true) + if err != nil { + return err + } + + log.Printf("PostQueueWorker: sent article %s in newsgroup %s to processArticle", article.MessageID, newsgroup) + return nil +} + +// parseNewsgroups parses a comma-separated list of newsgroups +func parseNewsgroups(newsgroupsStr string) []string { + if newsgroupsStr == "" { + return nil + } + + // Split by comma and clean up + parts := strings.Split(newsgroupsStr, ",") + var newsgroups []string + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + newsgroups = append(newsgroups, part) + } + } + return newsgroups +} diff --git a/internal/processor/analyze.go b/internal/processor/analyze.go index e8afb7d9..25b17ceb 100644 --- a/internal/processor/analyze.go +++ b/internal/processor/analyze.go @@ -496,7 +496,7 @@ func (proc *Processor) cacheFileExists(cacheFile string) bool { } // GetCachedMessageIDs returns cached message IDs for download optimization -func (proc *Processor) GetCachedMessageIDs(groupName string, startArticle, endArticle int64) ([]nntp.HeaderLine, error) { +func (proc *Processor) GetCachedMessageIDs(groupName string, startArticle, endArticle int64) ([]*nntp.HeaderLine, error) { providerName := "unknown" if proc.Pool.Backend != nil { providerName = proc.Pool.Backend.Provider.Name @@ -518,7 +518,7 @@ func (proc *Processor) GetCachedMessageIDs(groupName string, startArticle, endAr } defer file.Close() - var results []nntp.HeaderLine + var results []*nntp.HeaderLine scanner := bufio.NewScanner(file) for scanner.Scan() { @@ -550,7 +550,7 @@ func (proc *Processor) GetCachedMessageIDs(groupName string, startArticle, endAr continue } - results = append(results, nntp.HeaderLine{ + results = append(results, &nntp.HeaderLine{ ArticleNum: articleNum, Value: messageID, }) diff --git a/internal/processor/interface.go b/internal/processor/interface.go index dc7ac1a4..5c645ba7 100644 --- a/internal/processor/interface.go +++ b/internal/processor/interface.go @@ -1,5 +1,9 @@ package processor +import ( + "github.com/go-while/go-pugleaf/internal/database" +) + // MsgIdExists implements the ThreadingProcessor interface // Returns true if the message ID exists in the cache for the given group func (proc *Processor) MsgIdExists(group *string, messageID string) bool { @@ -10,3 +14,9 @@ func (proc *Processor) MsgIdExists(group *string, messageID string) bool { func (proc *Processor) IsNewsGroupInSectionsDB(name *string) bool { return proc.DB.IsNewsGroupInSections(*name) } + +// ForceCloseGroupDBs implements the ThreadingProcessor interface +// Forces closure of group database connections +func (proc *Processor) ForceCloseGroupDBs(groupsDB *database.GroupDBs) error { + return proc.DB.ForceCloseGroupDBs(groupsDB) +} diff --git a/internal/processor/proc_DLArt.go b/internal/processor/proc_DLArt.go index 06fa1088..94cf028d 100644 --- a/internal/processor/proc_DLArt.go +++ b/internal/processor/proc_DLArt.go @@ -61,8 +61,10 @@ func (bq *BatchQueue) GetOrCreateGroupBatch(newsgroup string) *GroupBatch { return groupBatch } +//var BatchItemDuplicateError = &BatchItem{Error: errIsDuplicateError} + // DownloadArticles fetches full articles and stores them in the articles DB. -func (proc *Processor) DownloadArticles(newsgroup string, ignoreInitialTinyGroups int64, DLParChan chan struct{}, progressDB *database.ProgressDB, start int64, end int64) error { +func (proc *Processor) DownloadArticles(newsgroup string, DLParChan chan struct{}, progressDB *database.ProgressDB, start int64, end int64, shutdownChan <-chan struct{}) error { //log.Printf("DEBUG-DownloadArticles: ng='%s' called with start=%d end=%d", newsgroup, start, end) DLParChan <- struct{}{} // aquire lock defer func() { @@ -98,24 +100,18 @@ func (proc *Processor) DownloadArticles(newsgroup string, ignoreInitialTinyGroup } //remaining := groupInfo.Last - end //log.Printf("DownloadArticles: Fetching XHDR for %s from %d to %d (last known: %d, remaining: %d)", newsgroup, start, end, groupInfo.Last, remaining) - var mux sync.Mutex - var lastGoodEnd int64 = 1 - toFetch := end - start + 1 // +1 because ranges are inclusive (start=1, end=3 means articles 1,2,3) - xhdrChan := make(chan *nntp.HeaderLine, MaxBatchSize) + var lastGoodEnd int64 = start + //toFetch := end - start + 1 // +1 because ranges are inclusive (start=1, end=3 means articles 1,2,3) + xhdrChan := make(chan *nntp.HeaderLine, 1000) errChan := make(chan error, 1) //log.Printf("Launch XHdrStreamed: '%s' toFetch=%d start=%d end=%d", newsgroup, toFetch, start, end) - go func(mux *sync.Mutex) { - aerr := proc.Pool.XHdrStreamed(newsgroup, "message-id", start, end, xhdrChan) - if aerr != nil { - log.Printf("Failed to fetch message IDs for group '%s': err='%v' toFetch=%d", newsgroup, aerr, toFetch) - errChan <- aerr - return - } - errChan <- nil - }(&mux) if proc.DB.IsDBshutdown() { - return fmt.Errorf("DownloadArticles: Database shutdown detected for group '%s'", newsgroup) + return fmt.Errorf("got shutdown in DownloadArticles: Database shutdown while in group '%s'", newsgroup) } + go func() { + errChan <- proc.Pool.XHdrStreamed(newsgroup, "message-id", start, end, xhdrChan, shutdownChan) + }() + //log.Printf("DownloadArticles: XHDR is fetching %d msgIds ng: '%s' (%d to %d)", len(messageIDs), newsgroup, start, end) releaseChan := make(chan struct{}, 1) notifyChan := make(chan int64, 1) @@ -124,6 +120,10 @@ func (proc *Processor) DownloadArticles(newsgroup string, ignoreInitialTinyGroup //log.Printf("DownloadArticles: Fetching %d articles for group '%s' using %d goroutines", toFetch, newsgroup, proc.Pool.Backend.MaxConns) var exists, queued int64 for hdr := range xhdrChan { + if proc.WantShutdown(shutdownChan) { + log.Printf("DownloadArticlesFromDate: Worker received shutdown signal, stopping") + return + } /* if !CheckMessageIdFormat(hdr.Value) { log.Printf("[FETCHER]: Invalid message ID format: '%s'", hdr.Value) @@ -150,8 +150,9 @@ func (proc *Processor) DownloadArticles(newsgroup string, ignoreInitialTinyGroup Batch.GetQ <- item // send to fetcher/main.go: for item := range processor.Batch.Queue queued++ //log.Printf("DownloadArticles: Queued article %d (%s) for group '%s'", hdr.ArticleNum, hdr.Value, *item.GroupName) - hdr.Value = "" - hdr.ArticleNum = 0 + //hdr.Value = "" + //hdr.ArticleNum = 0 + *hdr = nntp.HeaderLine{} } // end for xhdrChan //log.Printf("DownloadArticles: XHdr closed, finished feeding batch queue %d articles for group '%s' (existing: %d) total=%d", queued, newsgroup, exists, queued+exists) if queued == 0 { @@ -169,6 +170,10 @@ func (proc *Processor) DownloadArticles(newsgroup string, ignoreInitialTinyGroup deathCounter := 0 // Counter to track if we are stuck bulkmode := true var gotQueued int64 = -1 + if proc.WantShutdown(shutdownChan) { + log.Printf("DownloadArticlesFromDate: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } // Start processing loop forProcessing: for { @@ -214,7 +219,11 @@ forProcessing: case nntp.ErrArticleRemoved: notf++ default: - log.Printf("DownloadArticles: '%s' Error fetching article %s: %v .. continue", newsgroup, *item.MessageID, item.Error) + if item.MessageID != nil { + log.Printf("DownloadArticles: '%s' Error fetching article %s: %v .. continue", newsgroup, *item.MessageID, item.Error) + } else { + log.Printf("DownloadArticles: '%s' Error fetching item: %#v .. continue", newsgroup, item) + } errs++ } } else { @@ -248,7 +257,10 @@ forProcessing: } } } // end for processing routine (counts only) - + if proc.WantShutdown(shutdownChan) { + log.Printf("DownloadArticlesFromDate: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } if proc.DB.IsDBshutdown() { return fmt.Errorf("DownloadArticles: Database shutdown detected for group '%s'", newsgroup) } @@ -256,11 +268,16 @@ forProcessing: if xerr != nil { end = lastGoodEnd } - err = progressDB.UpdateProgress(proc.Pool.Backend.Provider.Name, newsgroup, end) - if err != nil { - log.Printf("Failed to update progress for provider '%s' group '%s': %v", proc.Pool.Backend.Provider.Name, newsgroup, err) + if gotQueued > 0 || dups > 0 { + // only update progress if we actually got something + err = progressDB.UpdateProgress(proc.Pool.Backend.Provider.Name, newsgroup, end) + if err != nil { + log.Printf("Failed to update progress for provider '%s' group '%s': %v", proc.Pool.Backend.Provider.Name, newsgroup, err) + } } - log.Printf("DownloadArticles: '%s' processed %d articles (dups: %d, gots: %d, errs: %d, added: %d) in %v end=%d", newsgroup, gots+errs+dups, dups, gots, errs, GroupCounter.GetReset(newsgroup), time.Since(startTime), end) + // threading.go:296: GroupCounter.Increment(newsgroup) // Increment the group counter + groupCnt := GroupCounter.GetReset(newsgroup) + log.Printf("DownloadArticles: '%s' processed %d articles [gotQueued=%d] (dups: %d, gots: %d, errs: %d, adds: %d) in %v end=%d", newsgroup, gots+errs+dups, gotQueued, dups, gots, errs, groupCnt, time.Since(startTime), end) // do another one if we haven't run enough times runtime.GC() @@ -272,40 +289,64 @@ forProcessing: // FindStartArticleByDate finds the first article number on or after the given date // using a simple binary search approach with XOVER data -func (proc *Processor) FindStartArticleByDate(groupName string, targetDate time.Time) (int64, error) { - // Get group info - groupInfo, err := proc.Pool.SelectGroup(groupName) - if err != nil { - return 0, fmt.Errorf("failed to select group: %w", err) +func (proc *Processor) FindStartArticleByDate(groupName string, targetDate time.Time, groupInfo *nntp.GroupInfo) (int64, error) { + /* + // Get group info + groupInfo, err := proc.Pool.SelectGroup(groupName) + if err != nil { + return 0, fmt.Errorf("failed to select group: %w", err) + } + */ + if groupInfo.Last == 0 { + log.Printf("No articles in group %s", groupName) + return 0, ErrIsEmptyGroup } - - first := groupInfo.First - last := groupInfo.Last + // decrease targetDate by 2 days + targetDate = targetDate.AddDate(0, 0, -2) log.Printf("Finding start article for date %s in group %s (range %d-%d)", - targetDate.Format("2006-01-02"), groupName, first, last) + targetDate.Format("2006-01-02"), groupName, groupInfo.First, groupInfo.Last) // Check if target date is before the first article enforceLimit := true - firstOverviews, err := proc.Pool.XOver(groupName, first, first, enforceLimit) + firstOverviews, err := proc.Pool.XOver(groupName, groupInfo.First, groupInfo.First, enforceLimit) if err == nil && len(firstOverviews) > 0 { firstArticleDate := ParseNNTPDate(firstOverviews[0].Date) if !firstArticleDate.IsZero() && targetDate.Before(firstArticleDate) { log.Printf("Target date %s is before first article %d (date: %s), returning first article. ng: %s", - targetDate.Format("2006-01-02"), first, firstArticleDate.Format("2006-01-02"), groupName) - return first, nil + targetDate.Format("2006-01-02"), groupInfo.First, firstArticleDate.Format("2006-01-02"), groupName) + return groupInfo.First, nil + } else { + log.Printf("Target date %s is after first article %d (date: %s), checking last article. ng: %s", + targetDate.Format("2006-01-02"), groupInfo.First, firstArticleDate.Format("2006-01-02"), groupName) + } + } + + // Check if target date is after the last article + lastOverviews, err := proc.Pool.XOver(groupName, groupInfo.Last, groupInfo.Last, enforceLimit) + if err == nil && len(lastOverviews) > 0 { + lastArticleDate := ParseNNTPDate(lastOverviews[0].Date) + if !lastArticleDate.IsZero() && targetDate.After(lastArticleDate) { + log.Printf("Target date %s is after last article %d (date: %s), returning last article. ng: %s", + targetDate.Format("2006-01-02"), groupInfo.Last, lastArticleDate.Format("2006-01-02"), groupName) + return groupInfo.Last, nil + } else { + log.Printf("Target date %s is before last article %d (date: %s), starting search. ng: %s", + targetDate.Format("2006-01-02"), groupInfo.Last, lastArticleDate.Format("2006-01-02"), groupName) } } + groupInfo.FetchStart = groupInfo.First + groupInfo.FetchEnd = groupInfo.Last // Binary search using 50% approach - for last-first > 1 { - mid := first + (last-first)/2 + for groupInfo.FetchEnd-groupInfo.FetchStart > 1 { + mid := groupInfo.FetchStart + (groupInfo.FetchEnd-groupInfo.FetchStart)/2 // Get XOVER for this article overviews, err := proc.Pool.XOver(groupName, mid, mid, enforceLimit) if err != nil || len(overviews) == 0 { // Article doesn't exist, try moving up - first = mid + groupInfo.FetchStart = mid continue } if proc.DB.IsDBshutdown() { @@ -313,35 +354,38 @@ func (proc *Processor) FindStartArticleByDate(groupName string, targetDate time. } articleDate := ParseNNTPDate(overviews[0].Date) if articleDate.IsZero() { - first = mid + groupInfo.FetchStart = mid continue } log.Printf("Scanning: %s - Article %d has date %s", groupName, mid, articleDate.Format("2006-01-02")) if articleDate.Before(targetDate) { - first = mid + groupInfo.FetchStart = mid } else { - last = mid + groupInfo.FetchEnd = mid } } - - log.Printf("Found start article: %d, ng: %s", last, groupName) - return last, nil + log.Printf("Found start article: %d, ng: %s", groupInfo.FetchEnd, groupName) + return groupInfo.FetchEnd, nil } +var ErrIsEmptyGroup = fmt.Errorf("isEmptyGroup") + // DownloadArticlesFromDate fetches articles starting from a specific date // Uses special progress tracking: sets progress to startArticle-1, or -1 if starting from article 1 // This prevents DownloadArticles from using "no progress detected" logic for existing groups -func (proc *Processor) DownloadArticlesFromDate(groupName string, startDate time.Time, ignoreInitialTinyGroups int64, DLParChan chan struct{}, progressDB *database.ProgressDB) error { +func (proc *Processor) DownloadArticlesFromDate(groupName string, startDate time.Time, DLParChan chan struct{}, progressDB *database.ProgressDB, groupInfo *nntp.GroupInfo, shutdownChan <-chan struct{}) error { //log.Printf("DownloadArticlesFromDate: Starting download from date %s for group '%s'", startDate.Format("2006-01-02"), groupName) // Find the starting article number based on date - startArticle, err := proc.FindStartArticleByDate(groupName, startDate) + startArticle, err := proc.FindStartArticleByDate(groupName, startDate, groupInfo) if err != nil { + if err == ErrIsEmptyGroup { + return err + } return fmt.Errorf("failed to find start article for date %s: %w", startDate.Format("2006-01-02"), err) } - // Open progress DB and temporarily override the last article position // so DownloadArticles will start from our desired article number // progressDB is now passed as parameter to avoid opening/closing for each group @@ -377,10 +421,12 @@ func (proc *Processor) DownloadArticlesFromDate(groupName string, startDate time //log.Printf("DownloadArticlesFromDate: Set progress to %d (date rescan), will start downloading from article %d", tempProgress, startArticle) // Get group info to calculate proper download range - groupInfo, err := proc.Pool.SelectGroup(groupName) - if err != nil { - return fmt.Errorf("failed to select group for date download: %w", err) - } + /* + groupInfo, err := proc.Pool.SelectGroup(groupName) + if err != nil { + return fmt.Errorf("failed to select group for date download: %w", err) + } + */ // Calculate download range: start from found article, end at current group last or startArticle + MaxBatch downloadStart := startArticle @@ -388,12 +434,19 @@ func (proc *Processor) DownloadArticlesFromDate(groupName string, startDate time if downloadEnd > groupInfo.Last { downloadEnd = groupInfo.Last } - + if proc.WantShutdown(shutdownChan) { + log.Printf("DownloadArticlesFromDate: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } //log.Printf("DownloadArticlesFromDate: Downloading range %d-%d for group '%s' (group last: %d)", downloadStart, downloadEnd, groupName, groupInfo.Last) // Now use the high-performance DownloadArticles function with proper article ranges - err = proc.DownloadArticles(groupName, ignoreInitialTinyGroups, DLParChan, progressDB, downloadStart, downloadEnd) + err = proc.DownloadArticles(groupName, DLParChan, progressDB, downloadStart, downloadEnd, shutdownChan) + if proc.WantShutdown(shutdownChan) { + log.Printf("DownloadArticlesFromDate: Worker received shutdown signal, stopping") + return fmt.Errorf("shutdown requested") + } // If there was an error and we haven't made progress, restore the original progress if err != nil && err != ErrUpToDate { // Check if we made any progress @@ -415,3 +468,15 @@ func (proc *Processor) DownloadArticlesFromDate(groupName string, startDate time //log.Printf("DownloadArticlesFromDate: Successfully completed download from date %s for group '%s'", startDate.Format("2006-01-02"), groupName) return err // Return the result from DownloadArticles (including ErrUpToDate) } + +func (proc *Processor) WantShutdown(shutdownChan <-chan struct{}) bool { + select { + case _, ok := <-shutdownChan: + if !ok { + // channel is closed + return true + } + default: + } + return false +} diff --git a/internal/processor/proc_DLArtOV.go b/internal/processor/proc_DLArtOV.go deleted file mode 100644 index 91f96f94..00000000 --- a/internal/processor/proc_DLArtOV.go +++ /dev/null @@ -1,176 +0,0 @@ -package processor - -import ( - "fmt" - "log" - "time" - - "github.com/go-while/go-pugleaf/internal/history" -) - -// DownloadArticlesViaOverview fetches full articles and stores them in the articles DB. -func (proc *Processor) DownloadArticlesViaOverview(groupName string) error { - - if err := proc.ImportOverview(groupName); err != nil { - return err - } - - groupDBs, err := proc.DB.GetGroupDBs(groupName) - if err != nil { - log.Printf("DownloadArticlesViaOverview: Failed to get group DBs for %s: %v", groupName, err) - return err - } - defer groupDBs.Return(proc.DB) - // Only fetch undownloaded overviews, in batches - undl, err := proc.DB.GetUndownloadedOverviews(groupDBs, int(MaxBatchSize)) - if err != nil { - return err - } - if len(undl) == 0 { - log.Printf("DownloadArticlesViaOverview: No undownloaded articles for group '%s'", groupName) - return nil - } - _, err = proc.Pool.SelectGroup(groupName) // Ensure remote has the group - if err != nil { - return fmt.Errorf("DownloadArticlesVO: Failed to select group '%s': %v", groupName, err) - } - log.Printf("proc.Pool.Backend=%#v", proc.Pool.Backend) - batchQueue := make(chan *BatchItem, len(undl)) - returnChan := make(chan *BatchItem, len(undl)) - // launch goroutines to fetch articles in parallel - runthis := int(float32(proc.Pool.Backend.MaxConns) * 0.75) // Use 75% of max connections for fetching articles - if runthis < 1 { - runthis = 1 // Ensure at least one goroutine runs - } - log.Printf("DownloadArticlesViaOverview: Fetching %d articles for group '%s' using %d goroutines", len(undl), groupName, runthis) - for i := 1; i <= runthis; i++ { - // Fetch article in a goroutines - go func(worker int) { - log.Printf("DownloadArticlesViaOverview: Worker %d group '%s' start", worker, groupName) - defer func() { - log.Printf("DownloadArticlesViaOverview: Worker %d group '%s' quit", worker, groupName) - }() - var processed int64 - for item := range batchQueue { - //log.Printf("DownloadArticlesViaOverview: Worker %d processing group '%s' article %d (%s)", worker, *item.GroupName, *item.ArticleNum, *item.MessageID) - art, err := proc.Pool.GetArticle(item.MessageID, true) - if err != nil { - log.Printf("DownloadArticlesViaOverview: group '%s' Failed to fetch article %s: %v", groupName, *item.MessageID, err) - item.Error = err // Set error on item - returnChan <- item // Send failed item back - return - } - processed++ - item.Article = art // set pointer - returnChan <- item // Send back the successfully imported article - } // end for item - log.Printf("DownloadArticlesViaOverview: Worker %d group '%s' processed %d articles", worker, groupName, processed) - }(i) - } // end for runthis anonymous go routines - - // for every undownloaded overview entry, create a batch item - //batchMap := make(map[int64]*BatchItem, len(undl)) - var batchList []*BatchItem - for _, ov := range undl { - msgID := ov.MessageID - num := ov.ArticleNum - /* - var exists int - if err := database.RetryableQueryRowScan(groupDBs.DB, "SELECT 1 FROM articles WHERE message_id = ? LIMIT 1", []interface{}{msgID}, &exists); err == nil { - // Already exists, mark as downloaded if not alread - if ov.Downloaded == 0 { - //err = proc.DB.SetOverviewDownloaded(groupDBs, num, 1) - if err != nil { - log.Printf("DownloadArticlesViaOverview: group '%s' Failed to mark article %d (%s) as downloaded: %v", groupName, num, msgID, err) - continue - } - log.Printf("DownloadArticlesViaOverview: group '%s' Marked article %d (%s) as downloaded", groupName, num, msgID) - - } - log.Printf("DownloadArticlesViaOverview: group '%s' Article %d (%s) already exists in articles DB, skipping import", groupName, num, msgID) - continue - } - */ - item := &BatchItem{ - MessageID: &msgID, - ArticleNum: num, - GroupName: &groupName, - // Article: nil, // will be set by the goroutine - } - batchQueue <- item // send to batch queue - batchList = append(batchList, item) // also add to batchList for later processing - //log.Printf("DownloadArticlesViaOverview: Queued article %d (%s) for group '%s'", num, msgID, groupName) - } // end for undl - - gots := 0 - errs := 0 - ticker := time.NewTicker(50 * time.Millisecond) - start := time.Now() - nextCheck := start.Add(5 * time.Second) - lastGots := 0 - lastErrs := 0 - deathCounter := 11 // Counter to track if we are stuck - // Start processing loop -forProcessing: - for { - select { - case <-ticker.C: - //log.Printf("DownloadArticlesViaOverview: group '%s' %d, gots: %d, errs: %d, batchQueue len: %d", groupName, deathCounter, gots, errs, len(batchQueue)) - // Periodically check if we have may be stuck - if gots+errs >= len(undl) { - log.Printf("DownloadArticlesViaOverview: group '%s' All %d (gots: %d, errs: %d) articles processed, closing batch channel", groupName, gots+errs, gots, errs) - close(batchQueue) // Close channel to stop goroutines - break forProcessing // Exit processing loop if all items are processed - } - if gots > lastGots || errs > lastErrs { - nextCheck = time.Now().Add(5 * time.Second) // Reset last check time - lastGots = gots - lastErrs = errs - deathCounter = 11 // Reset death counter on progress - } - if nextCheck.Before(time.Now()) { - // If we haven't made progress in 5 seconds, log a warning - log.Printf("DownloadArticlesViaOverview: group '%s' Stuck? %d articles processed (%d got, %d errs) in last 5 seconds", groupName, gots+errs, gots, errs) - nextCheck = time.Now().Add(5 * time.Second) // Reset last check time - deathCounter-- - } - if deathCounter <= 0 { - log.Printf("DownloadArticlesViaOverview: group '%s' Timeout... stopping import deathCounter=%d", groupName, deathCounter) - close(batchQueue) // Close channel to stop goroutines - return fmt.Errorf("DownloadArticlesViaOverview: group '%s' Timeout... %d articles processed (%d got, %d errs)", groupName, gots+errs, gots, errs) - } - - case item := <-returnChan: - if item.Error != nil { - log.Printf("DownloadArticlesViaOverview: group '%s' Error fetching article %s: %v", groupName, *item.MessageID, item.Error) - errs++ - } else { - //log.Printf("DownloadArticlesViaOverview: group '%s' fetched article %d (%s)", groupName, *item.ArticleNum, *item.MessageID) - gots++ - } - } - } - - // now loop over the batchList and insert articles - for _, item := range batchList { - if item == nil { - continue // Skip nil items (not fetched) - } - if item.Article == nil { - log.Printf("DownloadArticlesViaOverview: group '%s' Article %d (%s) was not fetched successfully, breaking import", groupName, item.ArticleNum, *item.MessageID) - return fmt.Errorf("internal/processor: DownloadArticlesViaOverview group '%s' article %d (%s) was not fetched successfully", groupName, item.ArticleNum, *item.MessageID) - } - bulkmode := true - response, err := proc.processArticle(item.Article, groupName, bulkmode) - if err != nil { - log.Printf("DownloadArticlesViaOverview: group '%s' Failed to process article %d (%s): %v", groupName, item.ArticleNum, *item.MessageID, err) - continue // Skip this item on error - } - if response == history.CasePass { - log.Printf("DownloadArticlesViaOverview: group '%s' imported article %d (%s)", groupName, item.ArticleNum, *item.MessageID) - } - - } - - return nil -} // end func DownloadArticlesViaOverview diff --git a/internal/processor/proc_DLXHDR.go b/internal/processor/proc_DLXHDR.go index f042ccb8..6b0dd584 100644 --- a/internal/processor/proc_DLXHDR.go +++ b/internal/processor/proc_DLXHDR.go @@ -3,7 +3,7 @@ package processor import "github.com/go-while/go-pugleaf/internal/nntp" // GetXHDR fetches XHDR data for a group -func (proc *Processor) GetXHDR(groupName string, header string, start, end int64) ([]nntp.HeaderLine, error) { +func (proc *Processor) GetXHDR(groupName string, header string, start, end int64) ([]*nntp.HeaderLine, error) { // Fetch XHDR data from NNTP server xhdrData, err := proc.Pool.XHdr(groupName, header, start, end) if err != nil { diff --git a/internal/processor/processor.go b/internal/processor/processor.go index df1686e6..86aed61b 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -34,19 +34,18 @@ type Processor struct { var ( // these list of ' var ' can be set after importing the lib before starting!! + MaxCrossPosts int = 15 // HARDCODED Maximum number of crossposts to allow per article + + LocalNNTPHostname string = "" // Hostname must be set before processing articles // MaxBatch defines the maximum number of articles to fetch in a single batch - MaxBatchSize int64 = 100 + MaxBatchSize int64 = 128 // UseStrictGroupValidation for group names, false allows upper-case in group names UseStrictGroupValidation = true - // must be set to true before booting and running DownloadArticlesViaOverview or ImportOverview! - XoverCopy = false - // RunRSLIGHTImport is used to indicate if the importer should run the legacy RockSolid Light importer RunRSLIGHTImport = false - DownloadMaxPar = 16 // Global Batch Queue (proc_DLArt.go) Batch = &BatchQueue{ @@ -68,12 +67,15 @@ var ( ) func NewProcessor(db *database.Database, nntpPool *nntp.Pool, useShortHashLen int) *Processor { - if LocalHostnamePath == "" { - log.Fatalf("FATAL: LocalHostnamePath is not set, please configure the hostname before starting the processor") + if LocalNNTPHostname == "" { + // Try to get hostname from database if not set + if err := SetHostname("", db); err != nil { + log.Fatalf("FATAL: LocalNNTPHostname is not set and could not retrieve from database: %v", err) + } } // Perform comprehensive hostname validation (similar to INN2) - if err := validateHostname(LocalHostnamePath); err != nil { + if err := validateHostname(LocalNNTPHostname); err != nil { log.Fatalf("FATAL: Hostname validation failed: %v", err) } // Ensure main database exists and is properly set up before any import operations @@ -300,19 +302,51 @@ func (proc *Processor) DisableBridges() { // SetHostname sets and validates the hostname for NNTP operations // This must be called before creating a new processor -func SetHostname(hostname string) error { - if err := validateHostname(hostname); err != nil { - return fmt.Errorf("hostname validation failed: %v", err) +// If hostname is empty, it will attempt to retrieve it from the database +// If hostname is provided, it will be saved to the database and used +func SetHostname(hostname string, db *database.Database) error { + if db == nil { + return fmt.Errorf("database is required for hostname operations") + } + + // If hostname is provided, validate and save it to database + if hostname != "" { + if err := validateHostname(hostname); err != nil { + return fmt.Errorf("hostname validation failed: %v", err) + } + + // Save hostname to database + if err := db.SetConfigValue("local_nntp_hostname", hostname); err != nil { + return fmt.Errorf("failed to save hostname to database: %v", err) + } + + LocalNNTPHostname = hostname + log.Printf("Hostname set, validated, and saved to database: %s", hostname) + return nil + } + + // If no hostname provided, try to get from database + dbHostname, err := db.GetConfigValue("local_nntp_hostname") + if err != nil { + return fmt.Errorf("failed to retrieve hostname from database: %v", err) + } + + if dbHostname == "" { + return fmt.Errorf("hostname is empty and no hostname configured in database") + } + + if err := validateHostname(dbHostname); err != nil { + return fmt.Errorf("database hostname validation failed: %v", err) } - LocalHostnamePath = hostname - log.Printf("Hostname set and validated: %s", hostname) + LocalNNTPHostname = dbHostname + log.Printf("Using hostname from database: %s", dbHostname) return nil } // GetHostname returns the currently configured hostname func GetHostname() string { - return LocalHostnamePath + return LocalNNTPHostname } // validateHostname performs comprehensive hostname validation similar to INN2 diff --git a/internal/processor/threading.go b/internal/processor/threading.go index a1c75330..eb0b9c7d 100644 --- a/internal/processor/threading.go +++ b/internal/processor/threading.go @@ -11,48 +11,6 @@ import ( "github.com/go-while/go-pugleaf/internal/models" ) -var MaxCrossPosts = 15 // HARDCODED Maximum number of crossposts to allow per article - -var LocalHostnamePath = "" // Hostname must be set before processing articles -/* -var ( - - processedArticleCount int64 - lastGCTime time.Time - gcMutex sync.Mutex - -) -*/ -const DefaultArticleItemPath = "no-path!not-for-mail" - -// Removed all object pools - they were causing race conditions and data corruption - -// Removed all pool functions - they were causing race conditions and data corruption -// Objects are now allocated normally and Go's GC handles cleanup - -// triggerGCIfNeeded forces garbage collection periodically during bulk imports to manage memory -func triggerGCIfNeeded(bulkmode bool) { - /* - if !bulkmode { - return // Only do this for bulk imports - } - - gcMutex.Lock() - defer gcMutex.Unlock() - - processedArticleCount++ - now := time.Now() - - // Force GC every 1000 articles or every 10 seconds, whichever comes first (made more aggressive for memory management) - - if processedArticleCount%1000 == 0 || now.Sub(lastGCTime) > 10*time.Second { - //runtime.GC() - lastGCTime = now - //log.Printf("[MEMORY] Forced GC after %d articles", processedArticleCount) - } - */ -} - // ComputeMessageIDHash computes MD5 hash of a message-ID func ComputeMessageIDHash(messageID string) string { hash := md5.Sum([]byte(messageID)) @@ -196,9 +154,8 @@ func (proc *Processor) processArticle(article *models.Article, legacyNewsgroup s return history.CaseError, fmt.Errorf("article '%s' posted too far in future: %v", article.MessageID, article.DateSent) } - // part of parsing data moved to nntp-client-commands.go:L~800 (func ParseLegacyArticleLines) + // part of parsing data moved to nntp-client-commands.go:L~850 (func ParseLegacyArticleLines) article.ReplyCount = 0 // Will be updated by threading - article.ImportedAt = time.Now() article.MsgIdItem = msgIdItem article.ArticleNums = make(map[*string]int64) article.ProcessQueue = make(chan *string, 16) // Initialize process queue @@ -224,9 +181,9 @@ func (proc *Processor) processArticle(article *models.Article, legacyNewsgroup s } if article.Path == "" { //log.Printf("[WARN:OLD] Article '%s' empty path... ?! headers='%#v'", article.MessageID, article.Headers) - article.Path = LocalHostnamePath + "!" + DefaultArticleItemPath + article.Path = LocalNNTPHostname + "!unknown!not-for-mail" } else { - article.Path = LocalHostnamePath + "!" + article.Path // Ensure path is prefixed with hostname + article.Path = LocalNNTPHostname + "!" + article.Path // Ensure path is prefixed with hostname } // Free memory from transient fields after extracting what we need @@ -310,8 +267,5 @@ func (proc *Processor) processArticle(article *models.Article, legacyNewsgroup s return history.CaseError, fmt.Errorf("error processArticle: article '%s' has no 'newsgroups' header", article.MessageID) } - // Memory optimization: trigger GC periodically during bulk imports - triggerGCIfNeeded(bulkmode) - return history.CasePass, nil } // end func processArticle diff --git a/internal/web/web_admin.go b/internal/web/web_admin.go index ead3916c..ef9620ca 100644 --- a/internal/web/web_admin.go +++ b/internal/web/web_admin.go @@ -22,35 +22,37 @@ type SpamArticleInfo struct { // AdminPageData represents data for admin page type AdminPageData struct { TemplateData - Users []*models.User - Newsgroups []*models.Newsgroup - NewsgroupPagination *models.PaginationInfo - NewsgroupSearch string - Providers []*models.Provider - APITokens []*database.APIToken - AIModels []*models.AIModel - NNTPUsers []*models.NNTPUser - SiteNews []*models.SiteNews // Added for site news management - Sections []*models.Section // Added for section management - SectionGroups []*models.SectionGroup // Added for section-newsgroup assignments - SpamArticles []*SpamArticleInfo // Added for spam management - SpamPagination *models.PaginationInfo // Added for spam pagination - CurrentUser *models.User - AdminCount int - EnabledTokensCount int - ActiveSessions int - ActiveNNTPUsers int - PostingNNTPUsers int - Uptime string - CacheStats map[string]interface{} // Added for cache monitoring - NewsgroupCacheStats map[string]interface{} // Added for newsgroup cache monitoring - ArticleCacheStats map[string]interface{} // Added for article cache monitoring - NNTPAuthCacheStats map[string]interface{} // Added for NNTP auth cache monitoring - MessageIdCacheStats map[string]interface{} // Added for message ID cache monitoring - RegistrationEnabled bool // Added for registration control - Success string - Error string - ActiveTab string // Added for tab state + Users []*models.User + Newsgroups []*models.Newsgroup + NewsgroupPagination *models.PaginationInfo + NewsgroupSearch string + Providers []*models.Provider + APITokens []*database.APIToken + AIModels []*models.AIModel + NNTPUsers []*models.NNTPUser + SiteNews []*models.SiteNews // Added for site news management + Sections []*models.Section // Added for section management + SectionGroups []*models.SectionGroup // Added for section-newsgroup assignments + SpamArticles []*SpamArticleInfo // Added for spam management + SpamPagination *models.PaginationInfo // Added for spam pagination + CurrentUser *models.User + AdminCount int + EnabledTokensCount int + ActiveSessions int + ActiveNNTPUsers int + PostingNNTPUsers int + Uptime string + CacheStats map[string]interface{} // Added for cache monitoring + NewsgroupCacheStats map[string]interface{} // Added for newsgroup cache monitoring + ArticleCacheStats map[string]interface{} // Added for article cache monitoring + NNTPAuthCacheStats map[string]interface{} // Added for NNTP auth cache monitoring + MessageIdCacheStats map[string]interface{} // Added for message ID cache monitoring + RegistrationEnabled bool // Added for registration control + CurrentHostname string // Added for NNTP hostname configuration + WebPostMaxArticleSize string // Added for web post size configuration + Success string + Error string + ActiveTab string // Added for tab state } // getUptime returns server uptime (placeholder) diff --git a/internal/web/web_adminPage.go b/internal/web/web_adminPage.go index f32716cb..c2cb60a2 100644 --- a/internal/web/web_adminPage.go +++ b/internal/web/web_adminPage.go @@ -338,37 +338,53 @@ func (s *WebServer) adminPage(c *gin.Context) { registrationEnabled = true // Default to enabled on error } + // Get current NNTP hostname from database + currentHostname, err := s.DB.GetConfigValue("local_nntp_hostname") + if err != nil { + log.Printf("Failed to get NNTP hostname: %v", err) + currentHostname = "" // Default to empty on error + } + + // Get current WebPostMaxArticleSize from database + webPostMaxSize, err := s.DB.GetConfigValue("WebPostMaxArticleSize") + if err != nil { + log.Printf("Failed to get WebPostMaxArticleSize: %v", err) + webPostMaxSize = "32768" // Default to 32KB on error + } + data := AdminPageData{ - TemplateData: s.getBaseTemplateData(c, "Admin Interface"), - Users: users, - Newsgroups: newsgroups, - NewsgroupPagination: newsgroupPagination, - NewsgroupSearch: searchTerm, - Providers: providers, - APITokens: apiTokens, - AIModels: aiModels, - NNTPUsers: nntpUsers, - SiteNews: siteNews, - Sections: sections, - SectionGroups: sectionGroups, - SpamArticles: spamArticles, - SpamPagination: spamPagination, - CurrentUser: currentUser, - AdminCount: s.countAdminUsers(users), - EnabledTokensCount: s.countEnabledAPITokens(apiTokens), - ActiveSessions: s.countActiveSessions(), - ActiveNNTPUsers: s.countActiveNNTPUsers(nntpUsers), - PostingNNTPUsers: s.countPostingNNTPUsers(nntpUsers), - Uptime: s.getUptime(), - CacheStats: cacheStats, - NewsgroupCacheStats: newsgroupCacheStats, - ArticleCacheStats: articleCacheStats, - NNTPAuthCacheStats: nntpAuthCacheStats, - MessageIdCacheStats: messageIdCacheStats, - RegistrationEnabled: registrationEnabled, - Success: session.GetSuccess(), - Error: session.GetError(), - ActiveTab: activeTab, // <-- add this field to AdminPageData struct + TemplateData: s.getBaseTemplateData(c, "Admin Interface"), + Users: users, + Newsgroups: newsgroups, + NewsgroupPagination: newsgroupPagination, + NewsgroupSearch: searchTerm, + Providers: providers, + APITokens: apiTokens, + AIModels: aiModels, + NNTPUsers: nntpUsers, + SiteNews: siteNews, + Sections: sections, + SectionGroups: sectionGroups, + SpamArticles: spamArticles, + SpamPagination: spamPagination, + CurrentUser: currentUser, + AdminCount: s.countAdminUsers(users), + EnabledTokensCount: s.countEnabledAPITokens(apiTokens), + ActiveSessions: s.countActiveSessions(), + ActiveNNTPUsers: s.countActiveNNTPUsers(nntpUsers), + PostingNNTPUsers: s.countPostingNNTPUsers(nntpUsers), + Uptime: s.getUptime(), + CacheStats: cacheStats, + NewsgroupCacheStats: newsgroupCacheStats, + ArticleCacheStats: articleCacheStats, + NNTPAuthCacheStats: nntpAuthCacheStats, + MessageIdCacheStats: messageIdCacheStats, + RegistrationEnabled: registrationEnabled, + CurrentHostname: currentHostname, + WebPostMaxArticleSize: webPostMaxSize, + Success: session.GetSuccess(), + Error: session.GetError(), + ActiveTab: activeTab, // <-- add this field to AdminPageData struct } // Load modular admin templates diff --git a/internal/web/web_admin_hostname.go b/internal/web/web_admin_hostname.go new file mode 100644 index 00000000..742e3103 --- /dev/null +++ b/internal/web/web_admin_hostname.go @@ -0,0 +1,54 @@ +package web + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-while/go-pugleaf/internal/processor" +) + +// adminSetHostname sets the NNTP hostname configuration +func (s *WebServer) adminSetHostname(c *gin.Context) { + // Check authentication and admin permissions + session := s.getWebSession(c) + if session == nil { + c.JSON(http.StatusUnauthorized, gin.H{"error": "Not authenticated"}) + return + } + + // Get current user + currentUser, err := s.DB.GetUserByID(int64(session.UserID)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to load user"}) + return + } + + // Check if user is admin + if !s.isAdmin(currentUser) { + c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"}) + return + } + + // Get the hostname from form + hostname := strings.TrimSpace(c.PostForm("hostname")) + + // Use SetHostname function to validate and save the hostname + // This will handle validation and database persistence + err = processor.SetHostname(hostname, s.DB) + if err != nil { + session.SetError("Failed to set hostname: " + err.Error()) + c.Redirect(http.StatusSeeOther, "/admin") + return + } + + // Set success message based on whether hostname was set or cleared + if hostname == "" { + session.SetSuccess("NNTP hostname cleared successfully") + } else { + session.SetSuccess("NNTP hostname set to: " + hostname) + } + + // Redirect back to admin page + c.Redirect(http.StatusSeeOther, "/admin") +} diff --git a/internal/web/web_admin_provider.go b/internal/web/web_admin_provider.go index 025e2f9a..9aeb0c83 100644 --- a/internal/web/web_admin_provider.go +++ b/internal/web/web_admin_provider.go @@ -1,6 +1,7 @@ package web import ( + "fmt" "log" "net/http" "strconv" @@ -39,6 +40,15 @@ func (s *WebServer) adminCreateProvider(c *gin.Context) { priorityStr := strings.TrimSpace(c.PostForm("priority")) maxArtSizeStr := strings.TrimSpace(c.PostForm("max_art_size")) enabledStr := c.PostForm("enabled") + postingEnabledStr := c.PostForm("posting_enabled") + + // Get proxy form data + proxyEnabledStr := c.PostForm("proxy_enabled") + proxyType := strings.TrimSpace(c.PostForm("proxy_type")) + proxyHost := strings.TrimSpace(c.PostForm("proxy_host")) + proxyPortStr := strings.TrimSpace(c.PostForm("proxy_port")) + proxyUsername := strings.TrimSpace(c.PostForm("proxy_username")) + proxyPassword := c.PostForm("proxy_password") // Validate required fields if name == "" || host == "" { @@ -97,6 +107,45 @@ func (s *WebServer) adminCreateProvider(c *gin.Context) { // Parse enabled status enabled := enabledStr == "on" || enabledStr == "true" + // Parse posting enabled status + postingEnabled := postingEnabledStr == "on" || postingEnabledStr == "true" + + // Parse proxy settings + proxyEnabled := proxyEnabledStr == "on" || proxyEnabledStr == "true" + + // Parse proxy port + proxyPort := 0 + if proxyPortStr != "" { + proxyPort, err = strconv.Atoi(proxyPortStr) + if err != nil || proxyPort < 0 || proxyPort > 65535 { + session.SetError("Invalid proxy port number") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + } + + // Validate proxy configuration if enabled + if proxyEnabled { + // Validate proxy type + if proxyType != "socks4" && proxyType != "socks5" { + session.SetError("Invalid proxy type. Must be socks4 or socks5") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + + // Validate host and port for proxy + if proxyHost == "" { + session.SetError("Proxy host is required for proxy type " + proxyType) + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + if proxyPort <= 0 || proxyPort > 65535 { + session.SetError(fmt.Sprintf("Invalid proxy port %d for proxy type %s", proxyPort, proxyType)) + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + } + // Check if provider already exists res, err := s.DB.GetProviderByName(name) if err != nil { @@ -108,18 +157,25 @@ func (s *WebServer) adminCreateProvider(c *gin.Context) { // Create provider provider := &models.Provider{ - Name: name, - Grp: grp, - Host: host, - Port: port, - SSL: ssl, - Username: username, - Password: password, - MaxConns: maxConns, - Priority: priority, - MaxArtSize: maxArtSize, - Enabled: enabled, - CreatedAt: time.Now(), + Name: name, + Grp: grp, + Host: host, + Port: port, + SSL: ssl, + Username: username, + Password: password, + MaxConns: maxConns, + Priority: priority, + MaxArtSize: maxArtSize, + Enabled: enabled, + Posting: postingEnabled, + ProxyEnabled: proxyEnabled, + ProxyType: proxyType, + ProxyHost: proxyHost, + ProxyPort: proxyPort, + ProxyUsername: proxyUsername, + ProxyPassword: proxyPassword, + CreatedAt: time.Now(), } err = s.DB.AddProvider(provider) @@ -164,6 +220,17 @@ func (s *WebServer) adminUpdateProvider(c *gin.Context) { priorityStr := strings.TrimSpace(c.PostForm("priority")) maxArtSizeStr := strings.TrimSpace(c.PostForm("max_art_size")) enabledStr := c.PostForm("enabled") + postingEnabledStr := c.PostForm("posting_enabled") + + // Get proxy form data + proxyEnabledStr := c.PostForm("proxy_enabled") + proxyType := strings.TrimSpace(c.PostForm("proxy_type")) + proxyHost := strings.TrimSpace(c.PostForm("proxy_host")) + proxyPortStr := strings.TrimSpace(c.PostForm("proxy_port")) + proxyUsername := strings.TrimSpace(c.PostForm("proxy_username")) + proxyPassword := c.PostForm("proxy_password") + clearProxyUsernameStr := c.PostForm("clear_proxy_username") + clearProxyPasswordStr := c.PostForm("clear_proxy_password") // Validate required fields if idStr == "" || name == "" || host == "" { @@ -173,7 +240,7 @@ func (s *WebServer) adminUpdateProvider(c *gin.Context) { } // Parse ID - id, err := strconv.Atoi(idStr) + id, err := strconv.ParseInt(idStr, 10, 64) if err != nil { session.SetError("Invalid provider ID") c.Redirect(http.StatusSeeOther, "/admin?tab=providers") @@ -230,6 +297,43 @@ func (s *WebServer) adminUpdateProvider(c *gin.Context) { // Parse enabled status enabled := enabledStr == "on" || enabledStr == "true" + // Parse posting enabled status + postingEnabled := postingEnabledStr == "on" || postingEnabledStr == "true" + + // Parse proxy settings + proxyEnabled := proxyEnabledStr == "on" || proxyEnabledStr == "true" + + // Parse proxy port + proxyPort := 0 + if proxyPortStr != "" { + proxyPort, err = strconv.Atoi(proxyPortStr) + if err != nil || proxyPort < 0 || proxyPort > 65535 { + session.SetError("Invalid proxy port number") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + } + + // Validate proxy configuration if enabled + if proxyEnabled { + if proxyHost == "" || proxyPort == 0 { + session.SetError("ERROR: Proxy host and port are required when proxy is enabled") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + if proxyPort <= 0 || proxyPort > 65535 { + session.SetError("ERROR: Invalid proxy port number") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + + if proxyType == "" { + session.SetError("ERROR: Proxy type is not set.") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + } + // Handle username and password clearing/preservation clearUsername := clearUsernameStr == "on" || clearUsernameStr == "true" clearPassword := clearPasswordStr == "on" || clearPasswordStr == "true" @@ -258,20 +362,69 @@ func (s *WebServer) adminUpdateProvider(c *gin.Context) { finalPassword = existingProvider.Password } + // Handle proxy credentials clearing/preservation + clearProxyUsername := clearProxyUsernameStr == "on" || clearProxyUsernameStr == "true" + clearProxyPassword := clearProxyPasswordStr == "on" || clearProxyPasswordStr == "true" + + // Handle proxy username: clear if checkbox is checked, otherwise use form value or preserve existing + finalProxyUsername := proxyUsername + if clearProxyUsername { + finalProxyUsername = "" + } else if proxyUsername == "" && existingProvider != nil { + finalProxyUsername = existingProvider.ProxyUsername + } + + // Handle proxy password: clear if checkbox is checked, otherwise use form value or preserve existing + finalProxyPassword := proxyPassword + if clearProxyPassword { + finalProxyPassword = "" + } else if proxyPassword == "" && existingProvider != nil { + finalProxyPassword = existingProvider.ProxyPassword + } + // Create provider struct for update provider := &models.Provider{ - ID: id, - Name: name, - Grp: grp, - Host: host, - Port: port, - SSL: ssl, - Username: finalUsername, - Password: finalPassword, - MaxConns: maxConns, - Priority: priority, - MaxArtSize: maxArtSize, - Enabled: enabled, + ID: id, + Name: name, + Grp: grp, + Host: host, + Port: port, + SSL: ssl, + Username: finalUsername, + Password: finalPassword, + MaxConns: maxConns, + Priority: priority, + MaxArtSize: maxArtSize, + Enabled: enabled, + Posting: postingEnabled, + ProxyEnabled: proxyEnabled, + ProxyType: proxyType, + ProxyHost: proxyHost, + ProxyPort: proxyPort, + ProxyUsername: finalProxyUsername, + ProxyPassword: finalProxyPassword, + } + + // Validate proxy configuration if enabled + if provider.ProxyEnabled { + // Validate proxy type + if proxyType != "socks4" && proxyType != "socks5" { + session.SetError("Invalid proxy type. Must be socks4 or socks5") + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + + // Validate host and port for proxy + if proxyHost == "" { + session.SetError("Proxy host is required for proxy type " + proxyType) + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } + if proxyPort <= 0 || proxyPort > 65535 { + session.SetError(fmt.Sprintf("Invalid proxy port %d for proxy type %s", proxyPort, proxyType)) + c.Redirect(http.StatusSeeOther, "/admin?tab=providers") + return + } } err = s.DB.SetProvider(provider) diff --git a/internal/web/web_admin_webpostsize.go b/internal/web/web_admin_webpostsize.go new file mode 100644 index 00000000..9540eb70 --- /dev/null +++ b/internal/web/web_admin_webpostsize.go @@ -0,0 +1,76 @@ +package web + +import ( + "net/http" + "strconv" + "strings" + + "github.com/gin-gonic/gin" +) + +// adminSetWebPostSize sets the WebPostMaxArticleSize configuration +func (s *WebServer) adminSetWebPostSize(c *gin.Context) { + // Check authentication and admin permissions + session := s.getWebSession(c) + if session == nil { + c.JSON(http.StatusUnauthorized, gin.H{"error": "Not authenticated"}) + return + } + + // Get current user + currentUser, err := s.DB.GetUserByID(int64(session.UserID)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to load user"}) + return + } + + // Check if user is admin + if !s.isAdmin(currentUser) { + c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"}) + return + } + + // Get the size from form + sizeStr := strings.TrimSpace(c.PostForm("size")) + + // Validate the size + if sizeStr == "" { + session.SetError("Size cannot be empty") + c.Redirect(http.StatusSeeOther, "/admin") + return + } + + size, err := strconv.Atoi(sizeStr) + if err != nil { + session.SetError("Invalid size format: must be a number") + c.Redirect(http.StatusSeeOther, "/admin") + return + } + + // Validate size range (minimum 1KB, maximum 1MB) + if size < 1024 { + session.SetError("Size must be at least 1024 bytes (1KB)") + c.Redirect(http.StatusSeeOther, "/admin") + return + } + + if size > 1048576 { + session.SetError("Size must not exceed 1048576 bytes (1MB)") + c.Redirect(http.StatusSeeOther, "/admin") + return + } + + // Save to database + err = s.DB.SetConfigValue("WebPostMaxArticleSize", sizeStr) + if err != nil { + session.SetError("Failed to save configuration: " + err.Error()) + c.Redirect(http.StatusSeeOther, "/admin") + return + } + + // Set success message + session.SetSuccess("Web post article size limit set to: " + sizeStr + " bytes") + + // Redirect back to admin page + c.Redirect(http.StatusSeeOther, "/admin") +} diff --git a/internal/web/web_ircPage.go b/internal/web/web_ircPage.go new file mode 100644 index 00000000..4ab3776d --- /dev/null +++ b/internal/web/web_ircPage.go @@ -0,0 +1,30 @@ +// Package web provides the HTTP server and web interface for go-pugleaf +package web + +import ( + "html/template" + "net/http" + + "github.com/gin-gonic/gin" +) + +// NewsPageData represents data for news page +type IRCPageData struct { + TemplateData +} + +// ircPage handles the "/SiteIRC" route to display IRC server information +func (s *WebServer) ircPage(c *gin.Context) { + data := IRCPageData{ + TemplateData: s.getBaseTemplateData(c, "IRC Server"), + } + + // Load template individually to avoid conflicts + tmpl := template.Must(template.ParseFiles("web/templates/base.html", "web/templates/irc.html")) + c.Header("Content-Type", "text/html") + err := tmpl.ExecuteTemplate(c.Writer, "base.html", data) + if err != nil { + s.renderError(c, http.StatusInternalServerError, "Template error", err.Error()) + return + } +} diff --git a/internal/web/web_newsPage.go b/internal/web/web_newsPage.go index 3b3cf946..d7410d45 100644 --- a/internal/web/web_newsPage.go +++ b/internal/web/web_newsPage.go @@ -13,7 +13,7 @@ type NewsPageData struct { TemplateData } -// newsPage handles the "/news" route to display site news +// newsPage handles the "/SiteNews" route to display site news func (s *WebServer) newsPage(c *gin.Context) { data := NewsPageData{ TemplateData: s.getBaseTemplateData(c, "Site News"), diff --git a/internal/web/web_sitePostPage.go b/internal/web/web_sitePostPage.go new file mode 100644 index 00000000..04d9c5f8 --- /dev/null +++ b/internal/web/web_sitePostPage.go @@ -0,0 +1,374 @@ +// Package web provides the HTTP server and web interface for go-pugleaf +package web + +import ( + "fmt" + "html/template" + "log" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-while/go-pugleaf/internal/models" + "github.com/go-while/go-pugleaf/internal/processor" + "github.com/go-while/go-pugleaf/internal/utils" +) + +// PostPageData represents data for posting page +type PostPageData struct { + TemplateData + PrefilledNewsgroup string + PrefilledSubject string + PrefilledBody string + Error string + Success string + WebPostMaxArticleSize string + IsReply bool + ReplyTo string + MessageID string + ReplyToArticleNum string + ReplyToMessageID string + ReplySubject string +} + +// sitePostPage handles the "/SitePost" route to display the posting form +func (s *WebServer) sitePostPage(c *gin.Context) { + // Check if user is authenticated + session := s.getWebSession(c) + if session == nil { + c.Redirect(http.StatusFound, "/login?redirect=/SitePost") + return + } + + // Get prefilled newsgroup from POST form data (from "New Thread" button) + prefilledNewsgroup := c.PostForm("newsgroup") + + // Check if this is a reply + replyToArticleNum := c.PostForm("reply_to") + replyToMessageID := c.PostForm("message_id") + isReply := replyToArticleNum != "" && replyToMessageID != "" + + var err error + article := &models.Article{} + if isReply { + // Get the original article to extract subject and body for reply + if articleNum, err := strconv.ParseInt(replyToArticleNum, 10, 64); err == nil { + // Get group database connection + if groupDBs, err := s.DB.GetGroupDBs(prefilledNewsgroup); err == nil { + defer groupDBs.Return(s.DB) + if reply_article, err := s.DB.GetArticleByNum(groupDBs, articleNum); err == nil { + // Handle subject with "Re: " prefix + if !strings.HasPrefix(strings.ToLower(reply_article.Subject), "re:") { + article.Subject = "Re: " + models.ConvertToUTF8(reply_article.Subject) + } + // Quote the original message body + if reply_article.BodyText != "" { + // Clean the body text first to remove HTML entities + cleanBodyText := models.ConvertToUTF8(reply_article.BodyText) + lines := strings.Split(cleanBodyText, "\n") + var quotedLines []string + + // Add header line with properly decoded FromHeader for NNTP posting + cleanFromHeader := models.ConvertToUTF8(reply_article.FromHeader) + quotedLines = append(quotedLines, fmt.Sprintf("On %s, %s wrote:", + reply_article.DateString, cleanFromHeader)) + quotedLines = append(quotedLines, "") + + // Quote each line with "> " + for _, line := range lines { + quotedLines = append(quotedLines, "> "+line) + } + + // Add empty lines for user's response + quotedLines = append(quotedLines, "", "") + + article.BodyText = strings.Join(quotedLines, "\n") + } + } else { + log.Printf("Warning: Failed to get article for reply: %v", err) + } + } else { + log.Printf("Warning: Failed to get group database for reply: %v", err) + } + } + } + + // Get max article size from database config + maxArticleSizeStr, err := s.DB.GetConfigValue("WebPostMaxArticleSize") + if err != nil { + log.Printf("Warning: Failed to get WebPostMaxArticleSize config in sitePostPage, using default: %v", err) + maxArticleSizeStr = "32768" // fallback to default + } + + // Create template data with no errors (this is just displaying the form) + pageTitle := "New Thread" + if isReply { + pageTitle = "Reply to" + } + var prefilledBodyStr string + if isReply && len(article.BodyText) > 0 { + // Use raw text for textarea - Go's html/template will automatically escape it + // This gives us clean text in the form while still being XSS-safe + prefilledBodyStr = article.BodyText + } + var prefilledSubjectStr string + if isReply && len(article.Subject) > 0 { + // Use raw text for input field - Go's html/template will automatically escape it + prefilledSubjectStr = article.Subject + } + data := PostPageData{ + TemplateData: s.getBaseTemplateData(c, pageTitle), + PrefilledNewsgroup: prefilledNewsgroup, + PrefilledSubject: prefilledSubjectStr, + PrefilledBody: prefilledBodyStr, + Error: "", // No errors when just displaying the form + Success: "", // No success message when just displaying the form + WebPostMaxArticleSize: maxArticleSizeStr, + IsReply: isReply, + ReplyToArticleNum: replyToArticleNum, + ReplyToMessageID: replyToMessageID, + ReplySubject: prefilledSubjectStr, + } + + // Load and render the posting form template + tmpl := template.Must(template.ParseFiles("web/templates/base.html", "web/templates/sitepost.html")) + c.Header("Content-Type", "text/html") + err = tmpl.ExecuteTemplate(c.Writer, "base.html", data) + if err != nil { + s.renderError(c, http.StatusInternalServerError, "Template error", err.Error()) + return + } +} + +// sitePostSubmit handles the POST submission of new articles from web interface +func (s *WebServer) sitePostSubmit(c *gin.Context) { + // Check if user is authenticated + session := s.getWebSession(c) + if session == nil { + c.Redirect(http.StatusFound, "/login") + return + } + // Get form data + subject := strings.TrimSpace(c.PostForm("subject")) + body := strings.TrimSpace(c.PostForm("body")) + newsgroupsStr := strings.TrimSpace(c.PostForm("newsgroups")) + //log.Printf("User %s is posting to newsgroups: %v, subject: %s", session.User.Username, newsgroupsStr, subject) + + // Check if this is a reply + replyTo := strings.TrimSpace(c.PostForm("reply_to")) + messageID := strings.TrimSpace(c.PostForm("message_id")) + isReply := replyTo != "" && messageID != "" + + // Get max article size from database config + maxArticleSizeStr, err := s.DB.GetConfigValue("WebPostMaxArticleSize") + if err != nil { + log.Printf("Warning: Failed to get WebPostMaxArticleSize config, using default: %v", err) + maxArticleSizeStr = "32768" // fallback to default + } + + maxArticleSize := 32768 // default fallback + if parsed, err := strconv.Atoi(maxArticleSizeStr); err == nil && parsed > 0 { + maxArticleSize = parsed + } + + // Validate required fields + var errors []string + if subject == "" { + errors = append(errors, "Subject is required") + } + if len(subject) > 72 { + errors = append(errors, "Subject must be less than 72 characters") + } + if body == "" { + errors = append(errors, "Message body is required") + } + if len(body) > maxArticleSize { + errors = append(errors, fmt.Sprintf("Message body must be less than %d bytes", maxArticleSize)) + } + if newsgroupsStr == "" { + errors = append(errors, "At least one newsgroup is required") + } + + // Parse newsgroups (space or comma separated) + var newsgroups []string + if newsgroupsStr != "" { + // Replace commas with spaces and split + newsgroupsStr = strings.ReplaceAll(newsgroupsStr, ",", " ") + parts := strings.FieldsSeq(newsgroupsStr) + for part := range parts { + if part != "" { + newsgroups = append(newsgroups, part) + } + } + } + + if len(newsgroups) == 0 { + errors = append(errors, "No valid newsgroups specified") + } + if len(newsgroups) > processor.MaxCrossPosts { + errors = append(errors, fmt.Sprintf("You can post to a maximum of %d newsgroups at once", processor.MaxCrossPosts)) + } + + if len(errors) == 0 { + // Validate that all newsgroups exist and are active + var validNewsgroups []string + for _, newsgroup := range newsgroups { + ng, err := s.DB.GetNewsgroupByName(newsgroup) + if err != nil { + errors = append(errors, fmt.Sprintf("Newsgroup '%s' does not exist", newsgroup)) + continue + } + if !ng.Active { + errors = append(errors, fmt.Sprintf("Newsgroup '%s' is not active for posting", newsgroup)) + continue + } + validNewsgroups = append(validNewsgroups, newsgroup) + } + // Use only valid newsgroups for further processing + newsgroups = validNewsgroups + // Check if we have any valid newsgroups after validation + if len(newsgroups) == 0 && len(errors) == 0 { + errors = append(errors, "No valid active newsgroups found") + } + } + + // Check if there are validation errors + if len(errors) > 0 { + data := PostPageData{ + TemplateData: s.getBaseTemplateData(c, "Posting failed"), + PrefilledNewsgroup: newsgroupsStr, + PrefilledSubject: subject, + PrefilledBody: body, + Error: strings.Join(errors, "; "), + WebPostMaxArticleSize: strconv.Itoa(maxArticleSize), + IsReply: isReply, + ReplyToArticleNum: replyTo, + ReplyToMessageID: messageID, + } + + tmpl := template.Must(template.ParseFiles("web/templates/base.html", "web/templates/sitepost.html")) + c.Header("Content-Type", "text/html") + err := tmpl.ExecuteTemplate(c.Writer, "base.html", data) + if err != nil { + s.renderError(c, http.StatusInternalServerError, "Template error", err.Error()) + } + return + } + var headers []string + headers = append(headers, "Newsgroups: "+strings.Join(newsgroups, ",")) + // Create article similar to threading.go + article := &models.Article{ + MessageID: generateMessageID(), + Subject: subject, + HeadersJSON: strings.Join(headers, "\n"), + FromHeader: fmt.Sprintf("%s <%s>", session.User.DisplayName, session.User.DisplayName+"@"+processor.LocalNNTPHostname), + DateString: time.Now().Format(time.RFC1123Z), + BodyText: body, + IsThrRoot: !isReply, // Only new threads are thread roots + IsReply: isReply, + Lines: strings.Count(body, "\n") + 1, + Bytes: len(body), + Path: ".POSTED!not-for-mail", + ArticleNums: make(map[*string]int64), + RefSlice: []string{}, + Headers: make(map[string][]string, 6), + } + article.Headers["newsgroups"] = []string{strings.Join(newsgroups, ",")} + article.Headers["subject"] = []string{subject} + article.Headers["from"] = []string{article.FromHeader} + article.Headers["date"] = []string{article.DateString} + article.Headers["message-id"] = []string{article.MessageID} + // If this is a reply, set up References header + if isReply { + log.Printf("Setting up References header for reply to message ID: %s", messageID) + // Try to find the original article to get its References + var originalRefs string + for _, newsgroup := range newsgroups { + groupDBs, err := s.DB.GetGroupDBs(newsgroup) + if err != nil { + log.Printf("Warning: Failed to get group DB for %s: %v", newsgroup, err) + continue + } + defer groupDBs.Return(s.DB) + + originalArticle, err := s.DB.GetArticleByMessageID(groupDBs, messageID) + if err != nil { + log.Printf("Warning: Failed to find original article %s in %s: %v", messageID, newsgroup, err) + continue + } + if originalArticle.References != "" { + originalRefs = originalArticle.References + } + break + } + + // Build new References header: original References + original Message-ID + article.References = strings.TrimSpace(originalRefs + " " + messageID) + article.RefSlice = utils.ParseReferences(article.References) + article.Headers["references"] = []string{article.References} + } + //log.Printf("Web posting: User '%s': article='%#v'", session.User.Username, article) + + // Put article into the queue channel + // This channel will be processed by the PostQueueWorker in the processor package + select { + case models.PostQueueChannel <- article: + log.Printf("Article queued successfully for user '%s', message-id: '%s'", session.User.Username, article.MessageID) + + default: + log.Printf("Warning: Post queue channel is full, article is lost.") + data := PostPageData{ + TemplateData: s.getBaseTemplateData(c, "New Thread"), + PrefilledNewsgroup: newsgroupsStr, + PrefilledSubject: subject, + PrefilledBody: body, + Error: "Server is busy, please try again later", + WebPostMaxArticleSize: strconv.Itoa(maxArticleSize), + IsReply: isReply, + ReplyToArticleNum: replyTo, + ReplyToMessageID: messageID, + } + + tmpl := template.Must(template.ParseFiles("web/templates/base.html", "web/templates/sitepost.html")) + c.Header("Content-Type", "text/html") + err := tmpl.ExecuteTemplate(c.Writer, "base.html", data) + if err != nil { + s.renderError(c, http.StatusInternalServerError, "Template error", err.Error()) + } + return + } + + // Show success page + successMsg := "Your message has been queued for posting. It will be processed shortly." + pageTitle := "New Thread" + if isReply { + successMsg = "Your reply has been queued for posting. It will be processed shortly." + pageTitle = "Reply to" + } + + data := PostPageData{ + TemplateData: s.getBaseTemplateData(c, pageTitle), + Success: successMsg, + WebPostMaxArticleSize: strconv.Itoa(maxArticleSize), + } + + tmpl := template.Must(template.ParseFiles("web/templates/base.html", "web/templates/sitepost.html")) + c.Header("Content-Type", "text/html") + err = tmpl.ExecuteTemplate(c.Writer, "base.html", data) + if err != nil { + s.renderError(c, http.StatusInternalServerError, "Template error", err.Error()) + return + } +} + +// generateMessageID creates a unique message ID for web-posted articles +func generateMessageID() string { + random, err := generateRandomHex(8) + if err != nil { + log.Printf("Error in generateMessageID: generating random hex: %v", err) + return fmt.Sprintf("<%d@%s>", time.Now().UnixNano(), processor.LocalNNTPHostname) + } + return fmt.Sprintf("<%d$%s@%s>", time.Now().UnixNano(), random, processor.LocalNNTPHostname) +} diff --git a/internal/web/webserver_core_routes.go b/internal/web/webserver_core_routes.go index ff4cac89..a9b7a67f 100644 --- a/internal/web/webserver_core_routes.go +++ b/internal/web/webserver_core_routes.go @@ -6,8 +6,8 @@ import ( "fmt" "html/template" "log" - "net/http" "net" + "net/http" "os" "strconv" "strings" @@ -392,9 +392,11 @@ func (s *WebServer) setupRoutes() { s.Router.POST("/admin/hierarchies/update", s.adminUpdateHierarchies) s.Router.POST("/admin/registration/enable", s.adminEnableRegistration) s.Router.POST("/admin/registration/disable", s.adminDisableRegistration) + s.Router.POST("/admin/hostname/set", s.adminSetHostname) + s.Router.POST("/admin/webpostsize/set", s.adminSetWebPostSize) // Legacy/admin routes (high priority - must come before dynamic routes) s.Router.GET("/", s.homePage) - s.Router.GET("/groups", s.groupsPage) // Legacy fallback + s.Router.GET("/groups", s.groupsPage) // groups listing s.Router.GET("/groups/", s.groupsPage) // Handle trailing slash s.Router.GET("/hierarchies", s.hierarchiesPage) // Hierarchies listing s.Router.GET("/hierarchies/", s.hierarchiesPage) // Handle trailing slash @@ -422,12 +424,17 @@ func (s *WebServer) setupRoutes() { s.Router.GET("/search/", s.searchPage) s.Router.GET("/stats", s.statsPage) s.Router.GET("/stats/", s.statsPage) - s.Router.GET("/help", s.helpPage) - s.Router.GET("/help/", s.helpPage) - s.Router.GET("/news", s.newsPage) // Site news page - s.Router.GET("/news/", s.newsPage) // Handle trailing slash - s.Router.GET("/sections", s.sectionsPage) // List all sections - s.Router.GET("/sections/", s.sectionsPage) // Handle trailing slash + s.Router.GET("/SiteHelp", s.helpPage) + s.Router.GET("/SiteHelp/", s.helpPage) + s.Router.GET("/SiteNews", s.newsPage) + s.Router.GET("/SiteNews/", s.newsPage) + s.Router.GET("/SiteIRC", s.ircPage) + s.Router.GET("/SiteIRC/", s.ircPage) + s.Router.GET("/SitePost", s.sitePostPage) + s.Router.POST("/SitePost", s.sitePostPage) + s.Router.POST("/SitePostSubmit", s.sitePostSubmit) // Handle form submission + s.Router.GET("/sections", s.sectionsPage) + s.Router.GET("/sections/", s.sectionsPage) // Demo and testing routes s.Router.GET("/demo/thread-tree", s.threadTreeDemoPage) @@ -480,9 +487,10 @@ func (s *WebServer) BotDetectionMiddleware() gin.HandlerFunc { for _, pattern := range badBots { if strings.Contains(strings.ToLower(userAgent), pattern) { // Log bot request - log.Printf("Bot blocked: %s from %s", userAgent, c.ClientIP()) - // You could block, throttle, or just log + log.Printf("Bot blocked: '%s' IP: %s", userAgent, c.ClientIP()) c.String(403, "403") + c.Abort() + return } } c.Next() @@ -634,26 +642,27 @@ func (s *WebServer) sectionValidationMiddleware() gin.HandlerFunc { // Skip validation for known non-section paths knownPaths := map[string]bool{ - "favicon.ico": true, - "robots.txt": true, - "static": true, - "admin": true, - "api": true, - "login": true, - "logout": true, - "register": true, - "profile": true, - "groups": true, - "hierarchies": true, - "hierarchy": true, - "search": true, - "stats": true, - "help": true, - "news": true, - "sections": true, - "demo": true, - "ping": true, - "aichat": true, + "favicon.ico": true, + "robots.txt": true, + "static": true, + "admin": true, + "api": true, + "login": true, + "logout": true, + "register": true, + "profile": true, + "groups": true, + "hierarchies": true, + "hierarchy": true, + "search": true, + "stats": true, + "SiteHelp": true, + "SiteNews": true, + "sections": true, + "demo": true, + "ping": true, + "aichat": true, + "hierarchy-groups": true, } if knownPaths[potentialSection] { diff --git a/internal/processor/deserialize-rslight-data-db-to-array-to-json.php b/php/deserialize-rslight-data-db-to-array-to-json.php similarity index 100% rename from internal/processor/deserialize-rslight-data-db-to-array-to-json.php rename to php/deserialize-rslight-data-db-to-array-to-json.php diff --git a/rsync.sh b/rsync.sh index 06a5218e..f21afa24 100755 --- a/rsync.sh +++ b/rsync.sh @@ -1,11 +1,14 @@ #!/bin/bash SERVERS=$(grep -vE "^#" ../config/go-pugleaf_ssh-server.txt) CDN=$(grep -vE "^#" ../config/go-pugleaf_cdn-server.txt|head -1) +UPDATEURL=$(grep -vE "^#" ../config/go-pugleaf_cdn-url.txt|head -1) echo "rsync update.tar.gz to $CDN" rsync -va --progress update.tar.gz "$CDN" while IFS="" read SERVER; do echo "rsync to $SERVER" #rsync -advz --delete-before build/ copynew.sh run_web*.sh "$SERVER":~/new/ & #sleep 0.3 + cp getUpdate.sh.template getUpdate.sh + sed "s/XXXURLXXX/$UPDATEURL/g" -i getUpdate.sh rsync -advz --progress web preload .update getUpdate.sh "$SERVER":~/ 1>/dev/null & done< <(echo "$SERVERS") diff --git a/web/templates/admin_providers.html b/web/templates/admin_providers.html index 8a62e909..1320d2ea 100644 --- a/web/templates/admin_providers.html +++ b/web/templates/admin_providers.html @@ -1,7 +1,306 @@ {{define "admin_providers"}}
- + +
+
+
+
+
+
📡 News Server
+ {{len .Providers}} providers +
+
+
+
+

+ Important: The Provider Name is directly linked to news fetching progress.
+ Changing the name will reset your progress!
+ When you've finished fetching news from an Archive Server:
+ • Disable the Archive Server
+ • Enable a pugleaf Server instead
+ All 'news-*.pugleaf.net' servers begin retention in September 2025.
+ Enable only your nearest pugleaf server for best performance!
+ pugleaf-fetcher will only fetch from first enabled provider! +

+
+
+
+
+ {{if .Providers}} +
+ + + + + + + + + + + + + + + + + {{range .Providers}} + + + + + + + + + + + + + + + + + + + {{end}} + +
UseEditPostHost:PortSSLCSizeDeletePrio
+ {{if .Enabled}} + ✅ + {{else}} + 🔴 + {{end}} + + {{.Name}} + + {{if .Posting}} + ✅ + {{else}} + ⚪ + {{end}} + {{.Host}}:{{.Port}} + {{if .SSL}} + ✅ + {{else}} + ⚪ + {{end}} + {{.MaxConns}}{{.MaxArtSize}} + {{if not .Enabled}} +
+
+ + +
+
+ {{else}} +
+ +
+ {{end}} + 		{{.Priority}}
+
+
+
Edit Provider: {{.Name}}
+
+
+
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+
+
+
+
+ + +
+
+ +
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+
+
+
+ + +
+ + +
+
+
+
+
+ + +
+ + +
+
+
+
+
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + bytes +
+
+
+
+ + +
+
+
🔒 Proxy Configuration
+
+
+
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+
+
+ + +
+ + +
+
+
+
+
+ + +
+ + +
+
+
+
+
+ + Tor: Use SOCKS5, host 127.0.0.1, port 9050. Supports .onion addresses in Host field.
+ +
+
+
+ + +
+
+
+
+
+ {{else}} +
+ No providers found. Create the first provider above. +
+ {{end}} +
+
+
+
+ +
@@ -11,6 +310,29 @@
🌐 Add News Server / Provider
+
+
+
+
+ + +
+
+
+
+
+
+ + +
+
+
+
+
@@ -19,6 +341,7 @@
🌐 Add News Server / Provider
placeholder="Provider name">
+
@@ -50,18 +374,8 @@
🌐 Add News Server / Provider
-
-
- -
- - -
-
-
+
@@ -77,6 +391,9 @@
🌐 Add News Server / Provider
placeholder="Optional">
+
+
+
@@ -96,197 +413,87 @@
🌐 Add News Server / Provider
+ +
+ + +
+
+
🔒 Proxy Configuration (Optional)
+ Configure SOCKS proxy settings. For Tor: use SOCKS5 with localhost:9050 +
+
+ +
+
+
+ +
+ + +
+
+
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + Tor Configuration: For Tor/Onion access, use SOCKS5 with host 127.0.0.1 and port 9050 (default Tor proxy).
+ .onion addresses: You can use .onion addresses directly in the Host field above when proxy is enabled.
+ +
+
+
+
-
- - -
-
-
-
-
-
🌐 Backend Management for fetching News
-

Important: The Provider Name is directly related to progress of fetching news. If you change the name later, you'll reset progress too!

- {{len .Providers}} providers -
-
-
- {{if .Providers}} -
- - - - - - - - - - - - - - - - - {{range .Providers}} - - - - - - - - - - - - - - - - - {{end}} - -
EditNameHost:PortSSLEnabledPriorityMax ConnsMax Art SizeCreatedDelete
- - {{.Name}}{{.Host}}:{{.Port}} - {{if .SSL}} - Yes - {{else}} - No - {{end}} - - {{if .Enabled}} - Enabled - {{else}} - Disabled - {{end}} - {{.Priority}}{{.MaxConns}}{{.MaxArtSize}}{{.CreatedAt.Format "2006-01-02 15:04"}} - {{if not .Enabled}} -
-
- - -
-
- {{else}} -
- -
- {{end}} -
-
-
-
Edit Provider: {{.Name}}
-
-
- -
-
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
-
-
-
- - -
- - -
-
-
-
-
- - -
- - -
-
-
-
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
- -
-
-
-
-
- {{else}} -
- No providers found. Create the first provider above. -
- {{end}} -
-
-
-
+ {{end}} diff --git a/web/templates/admin_statistics.html b/web/templates/admin_statistics.html index dccdb358..32f2c683 100644 --- a/web/templates/admin_statistics.html +++ b/web/templates/admin_statistics.html @@ -322,10 +322,6 @@
{{.CacheStats.size_human}}
- - - -
{{if .NNTPAuthCacheStats}} @@ -540,20 +536,54 @@
Registration Control
{{end}}
- - + {{end}} diff --git a/web/templates/article.html b/web/templates/article.html index eafc9467..6b916b61 100644 --- a/web/templates/article.html +++ b/web/templates/article.html @@ -71,9 +71,17 @@

{{.Article.PrintSanitized "subject" .GroupName}}

-
+
+
+ + + + +
-
+ @@ -87,12 +95,12 @@

{{.Article.PrintSanitized "subject" .GroupName}}

- {{end}}
+ {{end}} {{end}}
diff --git a/web/templates/base.html b/web/templates/base.html index a88a7470..74685a9b 100644 --- a/web/templates/base.html +++ b/web/templates/base.html @@ -107,11 +107,9 @@

🛠️ Admin {{end}} 🏠 Home - 📰 News - - 🗂️ Usenet + 🗂️ Hierarchies {{if .AvailableSections}} 📚 Sections {{end}} @@ -174,7 +172,8 @@

- Help + IRC + Help Icons Statistics Built with ❤️ by RockSolid fans diff --git a/web/templates/group.html b/web/templates/group.html index 7236cb2d..c942a6d6 100644 --- a/web/templates/group.html +++ b/web/templates/group.html @@ -36,10 +36,22 @@
-->
-
- - [ Threads ] - +
+
+ + [ Threads ] + +
+ {{if .User}} +
+
+ + +
+
+ {{end}}
diff --git a/web/templates/home.html b/web/templates/home.html index bac95144..580703ce 100644 --- a/web/templates/home.html +++ b/web/templates/home.html @@ -152,7 +152,7 @@

- Each network node has its own dedicated news section - visit Site News for node-specific updates + Each network node has its own dedicated news section - visit Site News for node-specific updates
@@ -171,89 +171,6 @@
--news--title--
--> - -
-
-
IRC Network online
- July 30, 2025 -
-

- 

-- irc.pugleaf.net Message of the Day -
-
-                  🚀 Welcome to PUGLEAF.NET IRC Network! 🚀
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-                    📡 Server: irc.pugleaf.net
-                    🔒 SSL/TLS: 6697 (SSL)
-                    ⚠️ Plain: 6667 (no SSL)
-                    🚀 Home: https://pugleaf.net
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-                    🛡️  NETWORK INFORMATION 🛡️
-
-        • This is a retro-themed IRC network focused on discussion,
-          nostalgia and the classic internet experience!
-
-        • Inspired by RockSolid Light and classic BBS systems!
-
-    🌍 TOR: ij7fmstcudrwyty5p6iloodeenur4wksovlhbyrfubat7eixplmrxvqd.onion
-       Port: 6667 (no SSL)
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-                    🏛️  MAIN CHANNELS 🏛️
-
-            #coding             - Code, development, and technical talk
-            #lounge             - General chat and hangout space
-            #retro              - Retro computing, vintage tech discussion
-            #rocksolid          - RIP Retro Guy memorial channel
-            #devnull            - Network support and assistance
-            #aichat             - Have fun with AI bots
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-                    📜 NETWORK RULES 📜
-
-            1. Be respectful and civil to all users
-            2. No spam, flooding, or excessive automated messages
-            3. No sharing of illegal content or malicious software
-            4. Follow standard IRC etiquette and common sense
-            5. Operators have final say in disputes
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-                    🔧 NETWORK SERVICES 🔧
-
-            • NickServ: Register and protect your nickname
-            • ChanServ: Channel registration and management
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-                    💡 USEFUL COMMANDS 💡
-
-            /LIST              - List all channels
-            /WHOIS       - Get information about a user
-            /JOIN #channel     - Join a channel
-            /MSG NickServ HELP - Get help with nickname services
-            /MOTD              - Display this message again
-
-            /nickserv identify <username> <password>
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-              🎯 "Bringing back the golden age of online communities" 🎯
-
-                   Thanks for connecting! Happy chatting!
-
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-
-

-
- -
@@ -313,11 +230,7 @@
**The go-pugleaf Development Team** *Honoring the past, building the future* -We are not responsible for this spam in news.admin.peering... --- - -*Cross-posted to relevant groups. -For technical updates,visit the web interface.

diff --git a/web/templates/irc.html b/web/templates/irc.html new file mode 100644 index 00000000..0f38bdd1 --- /dev/null +++ b/web/templates/irc.html @@ -0,0 +1,112 @@ +{{template "base.html" .}} + +{{define "content"}} +
+ +
+
+
+
+
+
+ go-pugleaf IRC Network +
+
+
+
+ +
+
+
+ +
+
+
IRC Network online
+ July 30, 2025 +
+

+ 

+- irc.pugleaf.net Message of the Day -
+
+                  🚀 Welcome to PUGLEAF.NET IRC Network! 🚀
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+                    📡 Server: irc.pugleaf.net
+                    🔒 SSL/TLS: 6697 (SSL)
+                    ⚠️ Plain: 6667 (no SSL)
+                    🚀 Home: https://pugleaf.net
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+                    🛡️  NETWORK INFORMATION 🛡️
+
+        • This is a retro-themed IRC network focused on discussion,
+          nostalgia and the classic internet experience!
+
+        • Inspired by RockSolid Light and classic BBS systems!
+
+    🌍 TOR: ij7fmstcudrwyty5p6iloodeenur4wksovlhbyrfubat7eixplmrxvqd.onion
+       Port: 6667 (no SSL)
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+                    🏛️  MAIN CHANNELS 🏛️
+
+            #coding             - Code, development, and technical talk
+            #lounge             - General chat and hangout space
+            #retro              - Retro computing, vintage tech discussion
+            #rocksolid          - RIP Retro Guy memorial channel
+            #devnull            - Network support and assistance
+            #aichat             - Have fun with AI bots
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+                    📜 NETWORK RULES 📜
+
+            1. Be respectful and civil to all users
+            2. No spam, flooding, or excessive automated messages
+            3. No sharing of illegal content or malicious software
+            4. Follow standard IRC etiquette and common sense
+            5. Operators have final say in disputes
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+                    🔧 NETWORK SERVICES 🔧
+
+            • NickServ: Register and protect your nickname
+            • ChanServ: Channel registration and management
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+                    💡 USEFUL COMMANDS 💡
+
+            /LIST              - List all channels
+            /WHOIS       - Get information about a user
+            /JOIN #channel     - Join a channel
+            /MSG NickServ HELP - Get help with nickname services
+            /MOTD              - Display this message again
+
+            /nickserv identify <username> <password>
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+              🎯 "Bringing back the golden age of online communities" 🎯
+
+                   Thanks for connecting! Happy chatting!
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+

+
+ + + + +
+
+
+
+
+
+{{end}} \ No newline at end of file diff --git a/web/templates/sitepost.html b/web/templates/sitepost.html new file mode 100644 index 00000000..90f35b27 --- /dev/null +++ b/web/templates/sitepost.html @@ -0,0 +1,180 @@ +{{template "base.html" .}} + +{{define "breadcrumb"}} + +{{end}} + +{{define "content"}} +
+
+
+
+
+ {{if .IsReply}} + Reply to + {{else}} + New Thread + {{end}} +
+
+
+ {{if .Error}} + + {{end}} + + {{if .Success}} + + {{else}} + +
+ {{if .IsReply}} + + + {{end}} + +
+ + +
+ Enter one or more newsgroup names separated by spaces or commas. +
Examples: comp.lang.go, alt.test misc.test +
+
+ +
+ + +
+ +
+ + +
+ Plain text only. No HTML formatting. + + 0 / {{.WebPostMaxArticleSize}} characters + + +
+
+ +
+ Cancel + +
+
+ + {{end}} +
+
+
+
+ +{{if not .Success}} +
+
+ +
+
+{{end}} + +{{if not .Success}} + +{{end}} + +{{end}} diff --git a/web/templates/thread.html b/web/templates/thread.html index 2d3366e0..829042c7 100644 --- a/web/templates/thread.html +++ b/web/templates/thread.html @@ -129,13 +129,6 @@
- {{if eq (index $message.ArticleNums $.GroupPtr) (index $.ThreadRoot.ArticleNums $.GroupPtr)}}
{{else}} @@ -164,9 +157,17 @@
-
+
+
+ + + + +
-
+ @@ -180,12 +181,12 @@
- {{end}}
+ {{end}} {{end}}
diff --git a/web/templates/threads.html b/web/templates/threads.html index 2bb602c1..eea981b9 100644 --- a/web/templates/threads.html +++ b/web/templates/threads.html @@ -60,7 +60,16 @@
Page {{.CurrentPage}} of {{.TotalPages}} • {{.TotalThreads}} total threads - + {{if .User}} +
+
+ + +
+
+ {{end}}