From e2ddff59eb371d9dd8ccfd97185f3a1f2f234028 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 13 Sep 2025 04:00:42 +0300 Subject: [PATCH 1/4] Initial commit with task details for issue #8 Adding CLAUDE.md with task information for AI processing. This file will be removed when the task is complete. Issue: https://github.com/linksplatform/Scripts/issues/8 --- CLAUDE.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..723d7f7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +Issue to solve: https://github.com/linksplatform/Scripts/issues/8 +Your prepared branch: issue-8-a29f360d +Your prepared working directory: /tmp/gh-issue-solver-1757725240169 + +Proceed. \ No newline at end of file From 15aee80ec032643874e67ca666b6a66c9bd5c278 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 13 Sep 2025 04:11:03 +0300 Subject: [PATCH 2/4] Update CloneAllOrganizationRepositoriesByHTTPS script to support pagination for organizations with more than 100 repositories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add fetch_all_repositories function with pagination support - Use per_page=100 and automatic page iteration to handle any number of repositories - Maintain backward compatibility with existing functionality - Add proper error handling for invalid JSON responses - Route diagnostic messages to stderr to avoid JSON corruption - Clean up temporary files after processing Fixes #8 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- ...CloneAllOrganizationRepositoriesByHTTPS.sh | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/Utils/CloneAllOrganizationRepositoriesByHTTPS.sh b/Utils/CloneAllOrganizationRepositoriesByHTTPS.sh index 499fa95..409d969 100755 --- a/Utils/CloneAllOrganizationRepositoriesByHTTPS.sh +++ b/Utils/CloneAllOrganizationRepositoriesByHTTPS.sh @@ -7,7 +7,59 @@ then exit 1 fi -repositories_json=$(curl -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$1/repos?per_page=200"); +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} clone() { clone_url=$(printf ${clone_url_with_double_quotes} | perl -pe 's~"(?.*)"~$+{clone_url}~g'); @@ -16,6 +68,9 @@ clone() { echo "Done cloning $clone_url_with_double_quotes." } +# Fetch all repositories using pagination +repositories_json=$(fetch_all_repositories "$1") + for clone_url_with_double_quotes in $( printf "${repositories_json}" | jq '.[] | .clone_url' ) do clone From 876e77f9d58af3032694c0e6a0d28060940336b3 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 13 Sep 2025 04:11:53 +0300 Subject: [PATCH 3/4] 'Auto-commit changes made by Claude MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude ' --- examples/comprehensive_test.sh | 91 +++++++++++++ examples/debug_api.sh | 12 ++ examples/test_clone_script.sh | 78 ++++++++++++ examples/test_clone_urls.sh | 78 ++++++++++++ examples/test_concat.sh | 17 +++ examples/test_final_function.sh | 127 +++++++++++++++++++ examples/test_fixed_clone_script.sh | 78 ++++++++++++ examples/test_fixed_clone_script_modified.sh | 72 +++++++++++ examples/test_fixed_script.sh | 91 +++++++++++++ examples/test_function.sh | 67 ++++++++++ examples/test_main_script.sh | 27 ++++ examples/test_pagination.sh | 72 +++++++++++ examples/test_pagination_simple.sh | 55 ++++++++ examples/test_real_script.sh | 16 +++ examples/test_repo_count.sh | 75 +++++++++++ examples/test_small_pages.sh | 52 ++++++++ examples/test_updated_function.sh | 59 +++++++++ 17 files changed, 1067 insertions(+) create mode 100755 examples/comprehensive_test.sh create mode 100755 examples/debug_api.sh create mode 100755 examples/test_clone_script.sh create mode 100755 examples/test_clone_urls.sh create mode 100755 examples/test_concat.sh create mode 100755 examples/test_final_function.sh create mode 100755 examples/test_fixed_clone_script.sh create mode 100755 examples/test_fixed_clone_script_modified.sh create mode 100755 examples/test_fixed_script.sh create mode 100755 examples/test_function.sh create mode 100755 examples/test_main_script.sh create mode 100755 examples/test_pagination.sh create mode 100755 examples/test_pagination_simple.sh create mode 100755 examples/test_real_script.sh create mode 100755 examples/test_repo_count.sh create mode 100755 examples/test_small_pages.sh create mode 100755 examples/test_updated_function.sh diff --git a/examples/comprehensive_test.sh b/examples/comprehensive_test.sh new file mode 100755 index 0000000..62576c9 --- /dev/null +++ b/examples/comprehensive_test.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +echo "=== Comprehensive Test of CloneAllOrganizationRepositoriesByHTTPS.sh ===" + +cd /tmp/gh-issue-solver-1757725240169 + +# Test 1: Verify pagination works with small page size +echo "" +echo "Test 1: Testing pagination with per_page=10" + +# Create a temporary modified version with small page size +cat > examples/test_small_pages.sh << 'EOF' +#!/bin/bash + +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=10 # Small page size to force pagination + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + rm -rf "$temp_dir" +} + +repositories_json=$(fetch_all_repositories "$1") +echo "Final count: $(echo "$repositories_json" | jq length)" >&2 +EOF + +chmod +x examples/test_small_pages.sh +./examples/test_small_pages.sh linksplatform 2>&1 | grep "page\|Total\|Final" + +# Test 2: Verify the main script works without cloning +echo "" +echo "Test 2: Testing main script logic (showing clone URLs instead of cloning)" + +# Create a version that shows what would be cloned +cp Utils/CloneAllOrganizationRepositoriesByHTTPS.sh examples/test_clone_urls.sh +sed -i 's/git clone --recurse-submodules -j8 ${clone_url};/echo " -> ${clone_url}";/' examples/test_clone_urls.sh +sed -i 's/echo "Cloning $clone_url_with_double_quotes..."/echo "Would clone $clone_url_with_double_quotes..."/' examples/test_clone_urls.sh +sed -i 's/echo "Done cloning $clone_url_with_double_quotes."/echo " Done."/' examples/test_clone_urls.sh + +echo "" +echo "Repository URLs that would be cloned (first 5):" +./examples/test_clone_urls.sh linksplatform 2>/dev/null | head -15 + +echo "" +echo "=== All Tests Passed! ===" +echo "" +echo "Summary:" +echo "- Pagination works correctly when there are multiple pages" +echo "- The script properly handles the current 85 repositories in linksplatform" +echo "- The script is ready for when the repository count exceeds 100" +echo "- All clone URLs are correctly extracted and formatted" \ No newline at end of file diff --git a/examples/debug_api.sh b/examples/debug_api.sh new file mode 100755 index 0000000..aa04a64 --- /dev/null +++ b/examples/debug_api.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "Testing API response..." +response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/linksplatform/repos?per_page=2&page=1") + +echo "Response length: ${#response}" +echo "First 200 characters:" +echo "$response" | head -c 200 +echo "" +echo "" +echo "Testing jq parsing:" +echo "$response" | jq length \ No newline at end of file diff --git a/examples/test_clone_script.sh b/examples/test_clone_script.sh new file mode 100755 index 0000000..7a782f3 --- /dev/null +++ b/examples/test_clone_script.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Make sure curl, perl and jq are installed + +if [ -z "$1" ] +then + echo "Organization name is required. It should be the first argument." + exit 1 +fi + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" + + while true; do + echo "Fetching page $page..." + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} + +clone() { + clone_url=$(printf ${clone_url_with_double_quotes} | perl -pe 's~"(?.*)"~$+{clone_url}~g'); + echo "Cloning $clone_url_with_double_quotes..." + echo "Would clone: ${clone_url}"; + echo "Done cloning $clone_url_with_double_quotes." +} + +# Fetch all repositories using pagination +repositories_json=$(fetch_all_repositories "$1") + +for clone_url_with_double_quotes in $( printf "${repositories_json}" | jq '.[] | .clone_url' ) +do + clone +done; +wait; diff --git a/examples/test_clone_urls.sh b/examples/test_clone_urls.sh new file mode 100755 index 0000000..133d589 --- /dev/null +++ b/examples/test_clone_urls.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Make sure curl, perl and jq are installed + +if [ -z "$1" ] +then + echo "Organization name is required. It should be the first argument." + exit 1 +fi + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} + +clone() { + clone_url=$(printf ${clone_url_with_double_quotes} | perl -pe 's~"(?.*)"~$+{clone_url}~g'); + echo "Would clone $clone_url_with_double_quotes..." + echo " -> ${clone_url}"; + echo " Done." +} + +# Fetch all repositories using pagination +repositories_json=$(fetch_all_repositories "$1") + +for clone_url_with_double_quotes in $( printf "${repositories_json}" | jq '.[] | .clone_url' ) +do + clone +done; +wait; diff --git a/examples/test_concat.sh b/examples/test_concat.sh new file mode 100755 index 0000000..e09cb24 --- /dev/null +++ b/examples/test_concat.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo "Testing array concatenation..." + +response1=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/linksplatform/repos?per_page=3&page=1") +response2=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/linksplatform/repos?per_page=3&page=2") + +echo "Response 1 length: $(echo "$response1" | jq length)" +echo "Response 2 length: $(echo "$response2" | jq length)" + +echo "Testing concatenation method 1:" +combined1=$(echo "$response1" "$response2" | jq -s 'add') +echo "Combined length method 1: $(echo "$combined1" | jq length)" + +echo "Testing concatenation method 2:" +combined2=$(jq -n --argjson r1 "$response1" --argjson r2 "$response2" '$r1 + $r2') +echo "Combined length method 2: $(echo "$combined2" | jq length)" \ No newline at end of file diff --git a/examples/test_final_function.sh b/examples/test_final_function.sh new file mode 100755 index 0000000..1881d68 --- /dev/null +++ b/examples/test_final_function.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +echo "Testing the final fetch_all_repositories function..." + +# Function to fetch all repositories with pagination support (copy of the final version) +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" + + while true; do + echo "Fetching page $page..." + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} + +# Test with small page size to verify pagination +echo "" +echo "=== Testing with small page size (per_page=10) ===" +temp_function() { + local org="$1" + local page=1 + local per_page=10 # Small page size to test pagination + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" + + while true; do + echo "Fetching page $page..." + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" + + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" >/dev/null 2>&1 # Suppress JSON output + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "Total repositories found: 0" >&2 + fi + + rm -rf "$temp_dir" +} + +temp_function "linksplatform" + +echo "" +echo "=== Testing with normal page size (per_page=100) ===" +repositories_json=$(fetch_all_repositories "linksplatform" 2>/dev/null) + +echo "" +echo "Repository count: $(echo "$repositories_json" | jq length)" + +echo "" +echo "First 3 repository names:" +echo "$repositories_json" | jq -r '.[0:3] | .[] | .name' + +echo "" +echo "Last 3 repository names:" +echo "$repositories_json" | jq -r '.[-3:] | .[] | .name' + +echo "" +echo "SUCCESS: Pagination is working correctly!" \ No newline at end of file diff --git a/examples/test_fixed_clone_script.sh b/examples/test_fixed_clone_script.sh new file mode 100755 index 0000000..409d969 --- /dev/null +++ b/examples/test_fixed_clone_script.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Make sure curl, perl and jq are installed + +if [ -z "$1" ] +then + echo "Organization name is required. It should be the first argument." + exit 1 +fi + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} + +clone() { + clone_url=$(printf ${clone_url_with_double_quotes} | perl -pe 's~"(?.*)"~$+{clone_url}~g'); + echo "Cloning $clone_url_with_double_quotes..." + git clone --recurse-submodules -j8 ${clone_url}; + echo "Done cloning $clone_url_with_double_quotes." +} + +# Fetch all repositories using pagination +repositories_json=$(fetch_all_repositories "$1") + +for clone_url_with_double_quotes in $( printf "${repositories_json}" | jq '.[] | .clone_url' ) +do + clone +done; +wait; diff --git a/examples/test_fixed_clone_script_modified.sh b/examples/test_fixed_clone_script_modified.sh new file mode 100755 index 0000000..b1000e6 --- /dev/null +++ b/examples/test_fixed_clone_script_modified.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Make sure curl, perl and jq are installed + +if [ -z "$1" ] +then + echo "Organization name is required. It should be the first argument." + exit 1 +fi + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} + +# Fetch all repositories using pagination +repositories_json=$(fetch_all_repositories "$1") + +# Test: just show the count and first few repository names +echo "Repository count: $(echo "$repositories_json" | jq length)" >&2 +echo "First 5 repositories:" >&2 +echo "$repositories_json" | jq -r '.[0:5] | .[] | .name' >&2 + +echo "Test completed successfully!" >&2 diff --git a/examples/test_fixed_script.sh b/examples/test_fixed_script.sh new file mode 100755 index 0000000..372a66b --- /dev/null +++ b/examples/test_fixed_script.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Test the fixed script + +echo "Testing the fixed script functionality..." + +cd /tmp/gh-issue-solver-1757725240169 + +# Create a test version that doesn't clone +cp Utils/CloneAllOrganizationRepositoriesByHTTPS.sh examples/test_fixed_clone_script.sh + +# Replace the clone function to just echo the first few URLs +cat > examples/test_fixed_clone_script_modified.sh << 'EOF' +#!/bin/bash +# Make sure curl, perl and jq are installed + +if [ -z "$1" ] +then + echo "Organization name is required. It should be the first argument." + exit 1 +fi + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Save this page to a temporary file + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + # Combine all pages into a single array + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + # Clean up temporary files + rm -rf "$temp_dir" +} + +# Fetch all repositories using pagination +repositories_json=$(fetch_all_repositories "$1") + +# Test: just show the count and first few repository names +echo "Repository count: $(echo "$repositories_json" | jq length)" >&2 +echo "First 5 repositories:" >&2 +echo "$repositories_json" | jq -r '.[0:5] | .[] | .name' >&2 + +echo "Test completed successfully!" >&2 +EOF + +chmod +x examples/test_fixed_clone_script_modified.sh + +echo "" +echo "Running fixed test script..." +./examples/test_fixed_clone_script_modified.sh linksplatform \ No newline at end of file diff --git a/examples/test_function.sh b/examples/test_function.sh new file mode 100755 index 0000000..caf2652 --- /dev/null +++ b/examples/test_function.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +echo "Testing the fetch_all_repositories function..." + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local all_repositories="" + + echo "Fetching repositories for organization: $org" + + while true; do + echo "Fetching page $page..." + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Append repositories to the collection + if [ -z "$all_repositories" ]; then + all_repositories="$response" + else + all_repositories=$(echo "$all_repositories" "$response" | jq -s 'add') + fi + + echo "Found $repo_count repositories on page $page" + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + local total_repos=$(echo "$all_repositories" | jq length) + echo "Total repositories found: $total_repos" + + echo "$all_repositories" +} + +repositories_json=$(fetch_all_repositories "linksplatform") + +echo "" +echo "Repository count: $(echo "$repositories_json" | jq length)" + +echo "" +echo "First 5 repository names:" +echo "$repositories_json" | jq -r '.[0:5] | .[] | .name' + +echo "" +echo "Last 5 repository names:" +echo "$repositories_json" | jq -r '.[-5:] | .[] | .name' + +echo "" +echo "Sample clone URLs:" +echo "$repositories_json" | jq -r '.[0:3] | .[] | .clone_url' \ No newline at end of file diff --git a/examples/test_main_script.sh b/examples/test_main_script.sh new file mode 100755 index 0000000..268e190 --- /dev/null +++ b/examples/test_main_script.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +echo "Testing the main script without actually cloning..." + +# Create a modified version that just lists repositories instead of cloning +cd /tmp/gh-issue-solver-1757725240169 + +# Test the fetch_all_repositories function +source Utils/CloneAllOrganizationRepositoriesByHTTPS.sh + +echo "Testing fetch_all_repositories function..." +repositories_json=$(fetch_all_repositories "linksplatform") + +echo "" +echo "Repository count: $(echo "$repositories_json" | jq length)" + +echo "" +echo "First 5 repository names:" +echo "$repositories_json" | jq -r '.[0:5] | .[] | .name' + +echo "" +echo "Last 5 repository names:" +echo "$repositories_json" | jq -r '.[-5:] | .[] | .name' + +echo "" +echo "Sample clone URLs:" +echo "$repositories_json" | jq -r '.[0:3] | .[] | .clone_url' \ No newline at end of file diff --git a/examples/test_pagination.sh b/examples/test_pagination.sh new file mode 100755 index 0000000..4de9a9d --- /dev/null +++ b/examples/test_pagination.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Test script to verify pagination functionality without actually cloning + +if [ -z "$1" ] +then + echo "Organization name is required. It should be the first argument." + exit 1 +fi + +# Function to fetch all repositories with pagination support (test version) +fetch_all_repositories_test() { + local org="$1" + local page=1 + local per_page=100 + local all_repositories="" + + echo "Fetching repositories for organization: $org" + + while true; do + echo "Fetching page $page..." + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Append repositories to the collection + if [ -z "$all_repositories" ]; then + all_repositories="$response" + else + all_repositories=$(echo "$all_repositories" "$response" | jq -s 'add') + fi + + echo "Found $repo_count repositories on page $page" + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + local total_repos=$(echo "$all_repositories" | jq length) + echo "Total repositories found: $total_repos" + + echo "$all_repositories" +} + +# Test with linksplatform organization +echo "Testing pagination with linksplatform organization..." +repositories_json=$(fetch_all_repositories_test "$1") + +# Count and display the first few repository names for verification +echo "" +echo "Repository count verification:" +echo "$repositories_json" | jq length + +echo "" +echo "First 5 repository names:" +echo "$repositories_json" | jq -r '.[0:5] | .[] | .name' + +echo "" +echo "Last 5 repository names:" +echo "$repositories_json" | jq -r '.[-5:] | .[] | .name' \ No newline at end of file diff --git a/examples/test_pagination_simple.sh b/examples/test_pagination_simple.sh new file mode 100755 index 0000000..d2c4d3b --- /dev/null +++ b/examples/test_pagination_simple.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +echo "Testing pagination with per_page=10 to verify it works..." + +org="linksplatform" +page=1 +per_page=10 +total_repos=0 + +echo "Fetching repositories for organization: $org" + +while true; do + echo "Fetching page $page..." + response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response on page $page, stopping" + break + fi + + repo_count=$(echo "$response" | jq length) + echo "Found $repo_count repositories on page $page" + + if [ "$repo_count" -eq 0 ]; then + echo "No more repositories found" + break + fi + + total_repos=$((total_repos + repo_count)) + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + echo "Last page reached (got $repo_count < $per_page)" + break + fi + + page=$((page + 1)) +done + +echo "" +echo "Total repositories found through pagination: $total_repos" + +# Compare with single request +echo "" +echo "Comparing with single request (per_page=200):" +single_response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=200") +single_count=$(echo "$single_response" | jq length) +echo "Single request found: $single_count repositories" + +if [ "$total_repos" -eq "$single_count" ]; then + echo "SUCCESS: Both methods found the same number of repositories!" +else + echo "ERROR: Different counts - pagination: $total_repos, single: $single_count" +fi \ No newline at end of file diff --git a/examples/test_real_script.sh b/examples/test_real_script.sh new file mode 100755 index 0000000..9803488 --- /dev/null +++ b/examples/test_real_script.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Test the actual script by modifying it to just show URLs instead of cloning + +echo "Testing the real script functionality..." + +cd /tmp/gh-issue-solver-1757725240169 + +# Create a test version that doesn't clone +cp Utils/CloneAllOrganizationRepositoriesByHTTPS.sh examples/test_clone_script.sh + +# Replace the clone function to just echo +sed -i 's/git clone --recurse-submodules -j8 ${clone_url};/echo "Would clone: ${clone_url}";/' examples/test_clone_script.sh + +echo "" +echo "Running test script with linksplatform..." +./examples/test_clone_script.sh linksplatform | head -20 \ No newline at end of file diff --git a/examples/test_repo_count.sh b/examples/test_repo_count.sh new file mode 100755 index 0000000..6f0eeb7 --- /dev/null +++ b/examples/test_repo_count.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +echo "Testing repository count with different methods..." + +echo "" +echo "1. Using the original method (single request with per_page=200):" +original_response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/linksplatform/repos?per_page=200") +original_count=$(echo "$original_response" | jq length) +echo "Original method found: $original_count repositories" + +echo "" +echo "2. Using pagination with per_page=100:" + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local all_repositories="" + local total_fetched=0 + + while true; do + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response on page $page, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + echo "No more repositories found on page $page" + break + fi + + # Append repositories to the collection + if [ -z "$all_repositories" ]; then + all_repositories="$response" + else + all_repositories=$(echo "$all_repositories" "$response" | jq -s 'add') + fi + + total_fetched=$((total_fetched + repo_count)) + echo "Page $page: found $repo_count repositories (total so far: $total_fetched)" + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + echo "Last page reached (got $repo_count < $per_page)" + break + fi + + page=$((page + 1)) + done + + local final_count=$(echo "$all_repositories" | jq length) + echo "Final count from pagination: $final_count repositories" + + echo "$all_repositories" +} + +repositories_json=$(fetch_all_repositories "linksplatform") + +echo "" +echo "3. Comparison:" +new_count=$(echo "$repositories_json" | jq length) +echo "Original method: $original_count repositories" +echo "New method: $new_count repositories" + +if [ "$new_count" -gt "$original_count" ]; then + echo "SUCCESS: New method found more repositories!" + echo "Difference: $((new_count - original_count)) additional repositories" +else + echo "Both methods found the same number of repositories." +fi \ No newline at end of file diff --git a/examples/test_small_pages.sh b/examples/test_small_pages.sh new file mode 100755 index 0000000..13cff46 --- /dev/null +++ b/examples/test_small_pages.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=10 # Small page size to force pagination + local temp_dir=$(mktemp -d) + local repo_files=() + + echo "Fetching repositories for organization: $org" >&2 + + while true; do + echo "Fetching page $page..." >&2 + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" >&2 + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + local page_file="${temp_dir}/page_${page}.json" + echo "$response" > "$page_file" + repo_files+=("$page_file") + + echo "Found $repo_count repositories on page $page" >&2 + + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + if [ ${#repo_files[@]} -gt 0 ]; then + jq -s 'add' "${repo_files[@]}" + local total_repos=$(jq -s 'add | length' "${repo_files[@]}") + echo "Total repositories found: $total_repos" >&2 + else + echo "[]" + echo "Total repositories found: 0" >&2 + fi + + rm -rf "$temp_dir" +} + +repositories_json=$(fetch_all_repositories "$1") +echo "Final count: $(echo "$repositories_json" | jq length)" >&2 diff --git a/examples/test_updated_function.sh b/examples/test_updated_function.sh new file mode 100755 index 0000000..491303a --- /dev/null +++ b/examples/test_updated_function.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +echo "Testing the updated fetch_all_repositories function..." + +# Function to fetch all repositories with pagination support +fetch_all_repositories() { + local org="$1" + local page=1 + local per_page=100 + local all_repositories="[]" + + echo "Fetching repositories for organization: $org" + + while true; do + echo "Fetching page $page..." + local response=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/orgs/$org/repos?per_page=$per_page&page=$page") + + # Check if the response is valid JSON and not empty + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "Invalid JSON response, stopping pagination" + break + fi + + local repo_count=$(echo "$response" | jq length) + if [ "$repo_count" -eq 0 ]; then + break + fi + + # Append repositories to the collection using jq + all_repositories=$(jq -n --argjson existing "$all_repositories" --argjson new "$response" '$existing + $new') + + echo "Found $repo_count repositories on page $page" + + # If we got fewer repositories than per_page, we've reached the end + if [ "$repo_count" -lt "$per_page" ]; then + break + fi + + page=$((page + 1)) + done + + local total_repos=$(echo "$all_repositories" | jq length) + echo "Total repositories found: $total_repos" + + echo "$all_repositories" +} + +repositories_json=$(fetch_all_repositories "linksplatform") + +echo "" +echo "Repository count: $(echo "$repositories_json" | jq length)" + +echo "" +echo "First 5 repository names:" +echo "$repositories_json" | jq -r '.[0:5] | .[] | .name' + +echo "" +echo "Last 5 repository names:" +echo "$repositories_json" | jq -r '.[-5:] | .[] | .name' \ No newline at end of file From c27db1bb3c2977ea7569931a3a8fd5cd3e03cf45 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 13 Sep 2025 04:11:55 +0300 Subject: [PATCH 4/4] Remove CLAUDE.md - Claude command completed --- CLAUDE.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 723d7f7..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,5 +0,0 @@ -Issue to solve: https://github.com/linksplatform/Scripts/issues/8 -Your prepared branch: issue-8-a29f360d -Your prepared working directory: /tmp/gh-issue-solver-1757725240169 - -Proceed. \ No newline at end of file